From b4979cb65b9ece5b7d77db26536b15594378c5fd Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 09:43:41 +0000 Subject: [PATCH 1/2] cultmesh-py: answer an unreadable RUDP frame and end the session after 8 consecutive refusals Heimdall's parity ruling. A frame that is no msgpack map used to be dropped silently; it is now answered message_invalid (field "message") and the session kept. MAX_CONSECUTIVE_REFUSED_FRAMES (8) consecutive unreadable frames, or consecutive deletes the write authority refuses, end the session after the capped frame is answered. The authority's decision is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-py/src/cultmesh_py/server.py | 45 ++++++++-- packages/cultmesh-py/tests/test_cultmesh.py | 89 +++++++++++++++++++ 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index 69583c1c..a871a84f 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -59,6 +59,13 @@ from cultnet_py.cultmesh_contracts import ( ) _LOGGER = logging.getLogger(__name__) + +# How many consecutive refused frames of one kind a RUDP session may send before it is ended: undecodable +# frames (reset by any frame that decodes) and deletes the write authority refuses (reset by any decoded +# frame that is not a refused delete). Fewer are answered with a typed refusal and the session is kept; the +# frame that reaches this number is answered too, then the session ends. The same number is the cap in every +# runtime. +MAX_CONSECUTIVE_REFUSED_FRAMES = 8 _EPHEMERAL_PAIR_ATTEMPTS = 64 @@ -90,6 +97,8 @@ class _DatabaseSubscription: class _RudpPeerConnection: session: CultNetRudpSession subscriptions: dict[str, _DatabaseSubscription] = field(default_factory=dict) + undecodable_frames_in_a_row: int = 0 + refused_deletes_in_a_row: int = 0 @dataclass @@ -390,6 +399,7 @@ class CultMeshLocalServer: if result.disconnected: peers.pop(remote_addr, None) continue + capped = False try: for delivered in result.delivered: if delivered.channel_id != "schema": @@ -397,15 +407,30 @@ class CultMeshLocalServer: try: message = msgpack.unpackb(delivered.payload, raw=False) except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): - continue - if not isinstance(message, dict): - continue - responses = self._handle_connection_message( - message, - peer.subscriptions, - over_rudp=True, - peer=CultMeshServerPeer(f"{remote_addr[0]}:{remote_addr[1]}", "rudp"), - ) + message = None + if isinstance(message, dict): + peer.undecodable_frames_in_a_row = 0 + responses = self._handle_connection_message( + message, + peer.subscriptions, + over_rudp=True, + peer=CultMeshServerPeer(f"{remote_addr[0]}:{remote_addr[1]}", "rudp"), + ) + refused_delete = message.get("schemaVersion") == "cultnet.document_delete.v0" and any( + response.get("code") == "write_not_authorized" for response in responses + ) + peer.refused_deletes_in_a_row = peer.refused_deletes_in_a_row + 1 if refused_delete else 0 + refused, run = refused_delete, peer.refused_deletes_in_a_row + else: + # A frame that is no CultNet message is answered, not dropped, and the session kept. + peer.undecodable_frames_in_a_row += 1 + refused, run = True, peer.undecodable_frames_in_a_row + responses = [self._message_invalid({}, "message", "CultMesh RUDP server could not read the message.")] + if refused and run >= MAX_CONSECUTIVE_REFUSED_FRAMES: + # The capped frame's refusal goes out with the goodbye; the session ends. + self._end_refused_peer(rudp_socket, peers, remote_addr, refusal=responses[0]) + capped = True + break for response in responses: if unsendable is None: unsendable = self._send_rudp_schema_frame( @@ -434,6 +459,8 @@ class CultMeshLocalServer: ) self._end_refused_peer(rudp_socket, peers, remote_addr, refusal=refusal) continue + if capped: + continue if unsendable is None and ( packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered ): diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index b49b6ed9..4844cd7c 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -24,7 +24,9 @@ from cultcache_py import ( define_document_type, ) from cultcache_py.interop import INTEROP_SCHEMA_VERSION, interop_note_document +from cultmesh_py.server import MAX_CONSECUTIVE_REFUSED_FRAMES from cultmesh_py.verify import verify +from cultmesh_py.write_authority import CultMeshRemoteWriteDecision from cultnet_py import ( compute_simulation_claim_hash, CultNetClientAuthorityScope, @@ -4889,6 +4891,93 @@ class CultMeshTests(unittest.TestCase): self.assertEqual(after["documents"], []) self.assertTrue(alive) + def _rudp_open(self, server: Any) -> Any: + return create_rudp_schema_transport( + host="127.0.0.1", + port=server.port, + connection_id=server.rudp_connection_id, + timeout_seconds=4.0, + max_fragment_bytes=1200, + ) + + def _rudp_answer(self, transport: Any, payload: bytes) -> dict[str, Any]: + transport.send("schema", payload) + return msgpack.unpackb(transport.receive(timeout_seconds=4.0).payload, raw=False) + + def _assert_rudp_session_ended(self, transport: Any, server: Any) -> None: + transport.send("schema", msgpack.packb(snapshot_request(message_id="after-cap").to_wire(), use_bin_type=True)) + with self.assertRaises(Exception): + transport.receive(timeout_seconds=1.0) + self.assertTrue(server._rudp_thread.is_alive()) + + def test_cultmesh_local_server_answers_unreadable_frames_and_ends_the_session_at_eight_in_a_row(self) -> None: + # An undecodable frame (not a msgpack map) is answered message_invalid and the session kept; + # eight in a row end it (the eighth is answered too); any decoded frame resets the run. + self.assertEqual(MAX_CONSECUTIVE_REFUSED_FRAMES, 8) + node = CultMesh.create_node(runtime_id="mesh-rudp-unreadable") + server = CultMesh.serve_node(node, write_authority=CultMeshGrantAllWrites()) + garbage = b"\xc1" + snapshot = msgpack.packb(snapshot_request(message_id="ok").to_wire(), use_bin_type=True) + transport = self._rudp_open(server) + try: + for _ in range(2): + for _ in range(MAX_CONSECUTIVE_REFUSED_FRAMES - 1): + reply = self._rudp_answer(transport, garbage) + self.assertEqual(reply["code"], "message_invalid") + self.assertEqual(reply["details"]["field"], "message") + self.assertEqual(self._rudp_answer(transport, snapshot)["messageId"], "ok") + for _ in range(MAX_CONSECUTIVE_REFUSED_FRAMES - 1): + self._rudp_answer(transport, garbage) + transport.send("schema", garbage) + eighth = msgpack.unpackb(transport.receive(timeout_seconds=4.0).payload, raw=False) + self.assertEqual(eighth["code"], "message_invalid") + self._assert_rudp_session_ended(transport, server) + finally: + transport.close() + server.stop() + + def test_cultmesh_local_server_ends_the_session_at_eight_refused_deletes_in_a_row(self) -> None: + class GrantOnlyKeyB: + def decide(self, request: Any) -> Any: + return CultMeshRemoteWriteDecision.grant() if request.record_key == "note:b" else CultMeshRemoteWriteDecision.refuse("no") + + document = self._rudp_bound_note() + node = CultMesh.create_node(runtime_id="mesh-rudp-deletes") + node.database.register_document(document) + server = CultMesh.serve_node(node, write_authority=GrantOnlyKeyB()) + + def delete(key: str) -> bytes: + return msgpack.packb({ + "schemaVersion": "cultnet.document_delete.v0", + "messageId": f"del-{key}", + "schemaId": document.schema_id, + "recordKey": key, + }, use_bin_type=True) + + snapshot = msgpack.packb(snapshot_request(message_id="ok").to_wire(), use_bin_type=True) + transport = self._rudp_open(server) + try: + def refused_run(count: int) -> None: + for _ in range(count): + reply = self._rudp_answer(transport, delete("note:a")) + self.assertEqual(reply["code"], "write_not_authorized") + self.assertEqual(reply["details"]["value"], "note:a") + + refused_run(MAX_CONSECUTIVE_REFUSED_FRAMES - 1) + # A granted delete resets the run (nothing is stored under it, so the store answers that). + self.assertEqual(self._rudp_answer(transport, delete("note:b"))["code"], "unregistered_document_delete") + refused_run(MAX_CONSECUTIVE_REFUSED_FRAMES - 1) + # So does any decoded non-delete frame. + self.assertEqual(self._rudp_answer(transport, snapshot)["messageId"], "ok") + refused_run(MAX_CONSECUTIVE_REFUSED_FRAMES - 1) + transport.send("schema", delete("note:a")) + eighth = msgpack.unpackb(transport.receive(timeout_seconds=4.0).payload, raw=False) + self.assertEqual(eighth["code"], "write_not_authorized") + self._assert_rudp_session_ended(transport, server) + finally: + transport.close() + server.stop() + def test_cultmesh_local_server_refuses_a_put_its_store_cannot_take_and_serves_on(self) -> None: # Well-formed puts the store refuses: a blank key, a payload that is not MessagePack, a # payload its schema cannot decode, a global document under another key. Each is refused -- 2.47.3 From bf816955371cae48187f4f82faf8e1624f7d7768 Mon Sep 17 00:00:00 2001 From: Libby Date: Sun, 8 Nov 2026 09:54:37 +0000 Subject: [PATCH 2/2] cultmesh-py: answer the capped frame reliably and end the session once the answer is acknowledged The 8th refusal was sent unreliable and unordered with only the first response, then the goodbye at once, so a lost datagram left the peer without it. All answers now go the normal reliable path; the session ends when the peer has acknowledged them (a lost one is resent) or after a 2 s grace, and serves nothing after the capped frame meanwhile. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-py/src/cultmesh_py/server.py | 37 +++++++++++---- packages/cultmesh-py/tests/test_cultmesh.py | 47 +++++++++++++++++++ 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index a871a84f..ab5267a7 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -66,6 +66,9 @@ _LOGGER = logging.getLogger(__name__) # frame that reaches this number is answered too, then the session ends. The same number is the cap in every # runtime. MAX_CONSECUTIVE_REFUSED_FRAMES = 8 + +# How long a session whose capped frame was answered waits for the peer to acknowledge the answer before it ends anyway. +_RUDP_CAPPED_ANSWER_GRACE_MS = 2000 _EPHEMERAL_PAIR_ATTEMPTS = 64 @@ -99,6 +102,8 @@ class _RudpPeerConnection: subscriptions: dict[str, _DatabaseSubscription] = field(default_factory=dict) undecodable_frames_in_a_row: int = 0 refused_deletes_in_a_row: int = 0 + # Set once the peer's capped frame has been answered: the session ends when the answers are acknowledged. + ending_deadline_ms: int | None = None @dataclass @@ -355,6 +360,7 @@ class CultMeshLocalServer: if packet.connection_id != self.rudp_connection_id: continue + self._end_answered_peers(rudp_socket, peers) now_ms = _now_ms() peer = peers.get(remote_addr) if packet.packet_type == CultNetRudpPacketType.CONNECT: @@ -399,11 +405,13 @@ class CultMeshLocalServer: if result.disconnected: peers.pop(remote_addr, None) continue - capped = False try: for delivered in result.delivered: if delivered.channel_id != "schema": continue + if peer.ending_deadline_ms is not None: + # The capped frame was answered; nothing after it is served while the answer is delivered. + break try: message = msgpack.unpackb(delivered.payload, raw=False) except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): @@ -426,11 +434,6 @@ class CultMeshLocalServer: peer.undecodable_frames_in_a_row += 1 refused, run = True, peer.undecodable_frames_in_a_row responses = [self._message_invalid({}, "message", "CultMesh RUDP server could not read the message.")] - if refused and run >= MAX_CONSECUTIVE_REFUSED_FRAMES: - # The capped frame's refusal goes out with the goodbye; the session ends. - self._end_refused_peer(rudp_socket, peers, remote_addr, refusal=responses[0]) - capped = True - break for response in responses: if unsendable is None: unsendable = self._send_rudp_schema_frame( @@ -440,6 +443,11 @@ class CultMeshLocalServer: msgpack.packb(response, use_bin_type=True), message_id=str(response.get("messageId") or ""), ) + if refused and run >= MAX_CONSECUTIVE_REFUSED_FRAMES: + # The capped frame is answered like any other, reliably. The session ends once the peer + # has acknowledged every answer (or the grace runs out): see _end_answered_peers. + peer.ending_deadline_ms = now_ms + _RUDP_CAPPED_ANSWER_GRACE_MS + break except Exception as error: # noqa: BLE001 - one peer's message never ends the thread # Whatever a peer's message makes the handling raise is that peer's failure. # The session recorded the frame's sequence, so it cannot be kept: a retransmit @@ -459,8 +467,6 @@ class CultMeshLocalServer: ) self._end_refused_peer(rudp_socket, peers, remote_addr, refusal=refusal) continue - if capped: - continue if unsendable is None and ( packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered ): @@ -472,11 +478,26 @@ class CultMeshLocalServer: if unsendable is not None: self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) + def _end_answered_peers( + self, + rudp_socket: socket.socket, + peers: dict[tuple[str, int], _RudpPeerConnection], + ) -> None: + """Ends each session whose capped frame was answered once the peer has acknowledged every answer (a lost + answer is resent first), or when the grace has run out.""" + now_ms = _now_ms() + for remote_addr, peer in list(peers.items()): + if peer.ending_deadline_ms is None: + continue + if peer.session.outstanding_reliable_packet_count == 0 or now_ms >= peer.ending_deadline_ms: + self._end_refused_peer(rudp_socket, peers, remote_addr) + def _poll_rudp_resends( self, rudp_socket: socket.socket, peers: dict[tuple[str, int], _RudpPeerConnection], ) -> None: + self._end_answered_peers(rudp_socket, peers) now_ms = _now_ms() for remote_addr, peer in list(peers.items()): for packet in peer.session.due_resends(now_ms): diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 4844cd7c..d96c4ab2 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -4936,6 +4936,53 @@ class CultMeshTests(unittest.TestCase): transport.close() server.stop() + def test_cultmesh_local_server_resends_a_lost_capped_answer_before_it_ends_the_session(self) -> None: + # The eighth refusal is answered reliably: the datagram carrying it is lost on the wire, and the + # peer still gets the answer (by resend) before the goodbye. An unreliable final answer, or a + # goodbye sent at once, would leave the peer without it. + node = CultMesh.create_node(runtime_id="mesh-rudp-lost-answer") + server = CultMesh.serve_node(node, write_authority=CultMeshGrantAllWrites()) + send = server._send_rudp_packet + dropped: list[Any] = [] + armed = threading.Event() + + def lossy(rudp_socket: Any, remote_addr: Any, packet: Any) -> Any: + if armed.is_set() and not dropped and packet.packet_type == CultNetRudpPacketType.DATA and b"message_invalid" in (packet.payload or b""): + dropped.append(packet) + return None + return send(rudp_socket, remote_addr, packet) + + server._send_rudp_packet = lossy + transport = self._rudp_open(server) + try: + for _ in range(MAX_CONSECUTIVE_REFUSED_FRAMES - 1): + self._rudp_answer(transport, b"\xc1") + armed.set() + reply = self._rudp_answer(transport, b"\xc1") + self.assertEqual(len(dropped), 1, "fixture: the first transmission of the last answer was lost") + self.assertEqual(reply["code"], "message_invalid") + self._assert_rudp_session_ended(transport, server) + finally: + transport.close() + server.stop() + + def test_cultmesh_local_server_serves_nothing_after_the_capped_frame(self) -> None: + node = CultMesh.create_node(runtime_id="mesh-rudp-after-cap") + server = CultMesh.serve_node(node, write_authority=CultMeshGrantAllWrites()) + transport = self._rudp_open(server) + try: + for _ in range(MAX_CONSECUTIVE_REFUSED_FRAMES - 1): + self._rudp_answer(transport, b"\xc1") + transport.send("schema", b"\xc1") + transport.send("schema", msgpack.packb(snapshot_request(message_id="behind-the-cap").to_wire(), use_bin_type=True)) + first = msgpack.unpackb(transport.receive(timeout_seconds=4.0).payload, raw=False) + self.assertEqual(first["code"], "message_invalid") + with self.assertRaises(Exception): + transport.receive(timeout_seconds=1.0) + finally: + transport.close() + server.stop() + def test_cultmesh_local_server_ends_the_session_at_eight_refused_deletes_in_a_row(self) -> None: class GrantOnlyKeyB: def decide(self, request: Any) -> Any: -- 2.47.3