diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index fe4edd52..413441c3 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import socket import threading import time @@ -20,6 +21,7 @@ from cultnet_py import ( create_tcp_framed_transport_profile, decode_rudp_packet, encode_rudp_packet, + is_permanent_send_error, wire_message_schema_descriptors, ) @@ -31,6 +33,8 @@ from cultnet_py.cultmesh_contracts import ( CultMeshVerseCatalog, ) +_LOGGER = logging.getLogger(__name__) + @dataclass(frozen=True) class _DatabaseSubscription: @@ -73,6 +77,16 @@ class CultMeshLocalServer: _stop: threading.Event = field(default_factory=threading.Event, init=False, repr=False) _thread: threading.Thread | None = field(default=None, init=False, repr=False) _rudp_thread: threading.Thread | None = field(default=None, init=False, repr=False) + _rudp_send_failures: int = field(default=0, init=False, repr=False) + + @property + def rudp_send_failures(self) -> int: + """Datagrams that could not be sent to a peer. Each is that peer's lost + datagram: a reliable packet stays pending and is resent, and no other + peer is affected. A datagram that can never be sent as built + (`is_permanent_send_error`) is not counted here: it ends that peer's + session, and the peer's goodbye names the error.""" + return self._rudp_send_failures def __post_init__(self) -> None: if self.max_snapshot_documents is not None and self.max_snapshot_documents < 0: @@ -208,7 +222,10 @@ class CultMeshLocalServer: while not self._stop.is_set(): try: wire, remote_addr = rudp_socket.recvfrom(65535) - except TimeoutError: + except (TimeoutError, ConnectionResetError): + # Windows reports an earlier ICMP port-unreachable, from a peer that + # has gone away, as an error on the next receive. It names no datagram + # of this read, so it is idle, not the end of the thread. self._poll_rudp_resends(rudp_socket, peers) continue except OSError: @@ -237,46 +254,74 @@ class CultMeshLocalServer: ) ) peers[remote_addr] = peer - self._send_rudp_packet( + unsendable = self._send_rudp_packet( rudp_socket, remote_addr, peer.session.accept_connect(packet, now_ms, b"cultmesh-local-rudp"), ) + if unsendable is not None: + self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) continue if peer is None: continue - result = peer.session.receive(packet, now_ms) - if result.reply is not None: - self._send_rudp_packet(rudp_socket, remote_addr, result.reply) - for ready in result.ready_to_send: - self._send_rudp_packet(rudp_socket, remote_addr, ready) + try: + result = peer.session.receive(packet, now_ms) + except ValueError: + # receive() has already recorded the packet's reliable sequence, so the + # session cannot be kept: a retransmit would be acknowledged and the + # frame silently lost. It ends, never the thread, and the peer is told. + self._end_refused_peer(rudp_socket, peers, remote_addr) + continue + unsendable: OSError | None = None + for outgoing in (*([result.reply] if result.reply is not None else []), *result.ready_to_send): + unsendable = self._send_rudp_packet(rudp_socket, remote_addr, outgoing) + if unsendable is not None: + break if result.disconnected: peers.pop(remote_addr, None) continue - for delivered in result.delivered: - if delivered.channel_id != "schema": - continue - try: - message = msgpack.unpackb(delivered.payload, raw=False) - except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): - continue - if not isinstance(message, dict): - continue - responses = self._handle_connection_message(message, peer.subscriptions) - for response in responses: - self._send_rudp_schema_frame( - rudp_socket, - remote_addr, - peer.session, - msgpack.packb(response, use_bin_type=True), - ) - if packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered: - self._send_rudp_packet( + try: + for delivered in result.delivered: + if delivered.channel_id != "schema": + continue + try: + message = msgpack.unpackb(delivered.payload, raw=False) + except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): + continue + if not isinstance(message, dict): + continue + responses = self._handle_connection_message(message, peer.subscriptions) + for response in responses: + if unsendable is None: + unsendable = self._send_rudp_schema_frame( + rudp_socket, + remote_addr, + peer.session, + msgpack.packb(response, use_bin_type=True), + ) + except Exception as error: # noqa: BLE001 - one peer's message never ends the thread + # Whatever a peer's message makes the handling raise is that peer's failure. + # The session recorded the frame's sequence, so it cannot be kept: a retransmit + # would be acknowledged and never handled. The session ends, never the thread, + # and the peer is told. The log names the error type only, because the error's + # own message can quote what the peer sent. + _LOGGER.warning( + "CultMesh RUDP server could not handle a peer's message (%s); that peer's session ended.", + type(error).__name__, + ) + self._end_refused_peer(rudp_socket, peers, remote_addr) + continue + if unsendable is None and ( + packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered + ): + unsendable = self._send_rudp_packet( rudp_socket, remote_addr, peer.session.create_ack_for_received(packet.sequence), ) + if unsendable is not None: + self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) def _poll_rudp_resends( self, @@ -286,7 +331,10 @@ class CultMeshLocalServer: now_ms = _now_ms() for remote_addr, peer in list(peers.items()): for packet in peer.session.due_resends(now_ms): - self._send_rudp_packet(rudp_socket, remote_addr, packet) + unsendable = self._send_rudp_packet(rudp_socket, remote_addr, packet) + if unsendable is not None: + self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) + break def _send_rudp_schema_frame( self, @@ -294,21 +342,62 @@ class CultMeshLocalServer: remote_addr: tuple[str, int], session: CultNetRudpSession, payload: bytes, - ) -> None: + ) -> OSError | None: for packet in session.send_many( "schema", payload, CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=_now_ms()), ): - self._send_rudp_packet(rudp_socket, remote_addr, packet) + unsendable = self._send_rudp_packet(rudp_socket, remote_addr, packet) + if unsendable is not None: + return unsendable + return None - @staticmethod def _send_rudp_packet( + self, rudp_socket: socket.socket, remote_addr: tuple[str, int], packet: CultNetRudpPacket, + ) -> OSError | None: + # A transient send failure is that peer's lost datagram: counted, never + # raised (raising here ended the RUDP thread for every peer). A permanent + # one (`is_permanent_send_error`) is returned: the datagram can never be + # sent as built, and the caller ends that peer's session. + wire = encode_rudp_packet(packet) + try: + rudp_socket.sendto(wire, remote_addr) + except OSError as error: + if is_permanent_send_error(error): + return error + self._rudp_send_failures += 1 + return None + + def _end_refused_peer( + self, + rudp_socket: socket.socket, + peers: dict[tuple[str, int], _RudpPeerConnection], + remote_addr: tuple[str, int], ) -> None: - rudp_socket.sendto(encode_rudp_packet(packet), remote_addr) + peer = peers.pop(remote_addr) + # The goodbye is built after the reset, or its ack field would acknowledge the + # very frame the session refused. + peer.session.reset_peer_state() + goodbye = peer.session.create_disconnect(b"session refused a packet") + self._send_rudp_packet(rudp_socket, remote_addr, goodbye) + + def _end_unsendable_peer( + self, + rudp_socket: socket.socket, + peers: dict[tuple[str, int], _RudpPeerConnection], + remote_addr: tuple[str, int], + error: OSError, + ) -> None: + """The session owes its peer a packet that can never be sent as built. It ends, + never the thread, and the peer's goodbye names the error.""" + peer = peers.pop(remote_addr, None) + if peer is None: + return + self._send_rudp_packet(rudp_socket, remote_addr, peer.session.end_unsendable_session(error)) def _handle_connection_message( self, diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 2b4f5bec..4c6afabc 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import json import os import socket @@ -47,10 +48,12 @@ from cultnet_py import ( CultNetSimulationConsensusCandidate, CultNetSimulationObservation, CultNetSimulationObservationHub, + CultNetRudpPacketType, CultNetRudpSocketMode, CultNetRudpSocketTransportConnection, CultNetRudpSocketTransportOptions, apply_shard_log_response, + decode_rudp_packet, document_delete, document_put_raw, hello, @@ -4529,5 +4532,459 @@ class CultMeshTests(unittest.TestCase): self.assertEqual(streams.latest_frame("mimir:kiyo-pro").sequence, 42) +class CultMeshRudpSendFailureTests(unittest.TestCase): + """One peer whose datagrams cannot be sent must not stop the RUDP thread + serving the others.""" + + def _client(self, server: Any, name: str) -> tuple[CultNetRudpSocketTransportConnection, tuple[str, int]]: + sock = bind_udp_socket() + client = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id=name, + socket=sock, + mode=CultNetRudpSocketMode.CLIENT, + connection_id=server.rudp_connection_id, + remote_addr=("127.0.0.1", server.port), + resend_delay_ms=25, + ) + ) + client.connect(b"join") + for _ in range(200): + client.receive_once() + if client.connected: + break + self.assertTrue(client.connected) + return client, sock.getsockname()[:2] + + def test_a_failed_send_to_one_peer_does_not_kill_the_rudp_thread(self) -> None: + import msgpack # type: ignore + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr): + raise OSError("injected send failure") + return original_sendto(sock, data, *args) + + request = hello(runtime_id="probe").to_bytes() + with patch.object(socket.socket, "sendto", sendto): + # The server's reply and ack to X both fail to send. + x.send("schema", request) + self._wait_for(lambda: server.rudp_send_failures > 0) + self.assertTrue(server._rudp_thread.is_alive()) + + y.send("schema", request) + response = None + for _ in range(200): + response = y.receive_once() + if response is not None: + break + time.sleep(0.005) + + self.assertIsNotNone(response, "Y is still served after X's send failed") + self.assertEqual(msgpack.unpackb(response.payload, raw=False)["runtimeId"], "mesh-server") + self.assertTrue(server._rudp_thread.is_alive()) + self.assertGreater(server.rudp_send_failures, 0) + finally: + server.stop() + + def _served(self, y: CultNetRudpSocketTransportConnection) -> Any: + import msgpack # type: ignore + + y.send("schema", hello(runtime_id="probe").to_bytes()) + for _ in range(200): + response = y.receive_once() + if response is not None: + return msgpack.unpackb(response.payload, raw=False) + time.sleep(0.005) + return None + + def test_a_vanished_peer_does_not_kill_the_rudp_thread(self) -> None: + # On Windows the closed port of a peer that went away is reported as + # ConnectionResetError (WinError 10054) by the next receive. It can only fail there. + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + x, _ = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + x.send("schema", hello(runtime_id="probe").to_bytes()) + x.socket.close() + time.sleep(1.0) # the server's reply and resends to X draw ICMP port-unreachable + + self.assertTrue(server._rudp_thread.is_alive()) + served = self._served(y) + self.assertIsNotNone(served, "Y is still served after X vanished") + self.assertEqual(served["runtimeId"], "mesh-server") + finally: + server.stop() + + def test_a_reset_reported_by_a_receive_is_idle_not_the_end_of_the_thread(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_recvfrom = socket.socket.recvfrom + resets = [3] + try: + y, _ = self._client(server, "peer-y") + + def recvfrom(sock: socket.socket, *args: Any) -> Any: + if sock is server._rudp_socket and resets[0] > 0: + resets[0] -= 1 + raise ConnectionResetError(errno.ECONNRESET, "injected reset") + return original_recvfrom(sock, *args) + + with patch.object(socket.socket, "recvfrom", recvfrom): + self._wait_for(lambda: resets[0] == 0) + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y)) + finally: + server.stop() + + def test_a_permanent_send_failure_ends_only_that_peers_session_and_the_goodbye_names_it(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + failures = [1] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr) and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # The server's response to X can never be sent as built. + x.send("schema", hello(runtime_id="probe").to_bytes()) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertIsNotNone(x.disconnect_reason, "X is told its session ended") + self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: ")) + self.assertGreater(len(x.disconnect_reason), len(b"packet could not be sent: ")) + self.assertTrue(server._rudp_thread.is_alive()) + + # Y is unaffected. + self.assertFalse(x.connected) + self.assertIsNotNone(self._served(y)) + self.assertEqual(server.rudp_send_failures, 0) + finally: + server.stop() + + def test_healthy_sends_are_not_failures(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + x, _ = self._client(server, "peer-x") + self.assertIsNotNone(self._served(x)) + self.assertEqual(server.rudp_send_failures, 0) + finally: + server.stop() + + def test_a_permanent_failure_in_a_resend_ends_only_that_peers_session(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + mode = ["transient"] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr) and mode[0] == "transient": + raise OSError(errno.ENOBUFS, "injected transient failure") + if args and tuple(args[-1]) == tuple(x_addr) and mode[0] == "permanent": + mode[0] = "healthy" + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # X's response is lost, so a resend is pending; then it can never be sent as built. + x.send("schema", hello(runtime_id="probe").to_bytes()) + self._wait_for(lambda: server.rudp_send_failures > 0) + self.assertFalse(x.disconnect_reason, "a transient failure ends no session") + mode[0] = "permanent" + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertIsNotNone(x.disconnect_reason) + self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: ")) + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y)) + finally: + server.stop() + + def test_a_permanent_failure_acknowledging_a_frame_ends_the_session(self) -> None: + import msgpack # type: ignore + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + failures = [1] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr) and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # Not a message the server answers, so its ack is the only datagram owed. + x.send("schema", msgpack.packb(1)) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertIsNotNone(x.disconnect_reason) + self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: ")) + self.assertTrue(server._rudp_thread.is_alive()) + finally: + server.stop() + + def test_a_connect_whose_accept_can_never_be_sent_starts_no_session(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + y, _ = self._client(server, "peer-y") + z_socket = bind_udp_socket() + z_addr = z_socket.getsockname()[:2] + failures = [1] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(z_addr) and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + z = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="peer-z", + socket=z_socket, + mode=CultNetRudpSocketMode.CLIENT, + connection_id=server.rudp_connection_id, + remote_addr=("127.0.0.1", server.port), + resend_delay_ms=25, + ) + ) + z.connect(b"join") + # A session that started would resend its Accept, and the resend would get through. + time.sleep(0.5) + z_socket.settimeout(0.05) + received = [] + while True: + try: + wire, _ = z_socket.recvfrom(65535) + except TimeoutError: + break + received.append(decode_rudp_packet(wire).packet_type) + self.assertEqual(received, [CultNetRudpPacketType.DISCONNECT]) + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y)) + finally: + server.stop() + + def test_a_packet_the_session_refuses_ends_only_that_peers_session(self) -> None: + from cultnet_py import CultNetRudpPacket, encode_rudp_packet + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + x, _ = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + # A fragment whose metadata the session refuses: fragment id 0 names no set. + refused = CultNetRudpPacket( + packet_type=CultNetRudpPacketType.DATA, + connection_id=server.rudp_connection_id, + sequence=0, + ack=0, + ack_mask=0, + channel_id="schema", + reliable=False, + fragment_id=0, + fragment_index=0, + fragment_count=2, + payload=b"x", + ) + x.socket.sendto(encode_rudp_packet(refused), ("127.0.0.1", server.port)) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertEqual(x.disconnect_reason, b"session refused a packet") + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y), "Y is still served") + finally: + server.stop() + + def test_a_message_whose_handling_fails_ends_only_that_peers_session(self) -> None: + import msgpack # type: ignore + + canary = "canary-7f3e1c" + # Well-formed frames whose fields have the wrong type: handling raises TypeError. + frames = [ + {"schemaVersion": "cultnet.database_subscribe.v0", "subscriptionId": canary, "schemaIds": 5}, + {"schemaVersion": "cultnet.database_subscribe.v0", "subscriptionId": canary, "recordKeys": True}, + ] + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + y, _ = self._client(server, "peer-y") + for frame in frames: + with self.subTest(frame=frame): + x, _ = self._client(server, "peer-x") + with self.assertLogs("cultmesh_py.server", level="WARNING") as logs: + x.send("schema", msgpack.packb(frame, use_bin_type=True)) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertEqual(x.disconnect_reason, b"session refused a packet") + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y), "Y is still served") + self.assertEqual(len(logs.output), 1) + self.assertIn("TypeError", logs.output[0]) + self.assertNotIn(canary, logs.output[0]) + finally: + server.stop() + + def _raw_peer(self, server: Any) -> tuple[socket.socket, Any]: + """A peer driven packet by packet, to deliver frames in an order a client would not.""" + from cultnet_py import CultNetRudpSession, CultNetRudpSessionOptions, encode_rudp_packet + + sock = bind_udp_socket() + sock.settimeout(0.05) + session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=server.rudp_connection_id)) + sock.sendto(encode_rudp_packet(session.create_connect(0, b"join")), ("127.0.0.1", server.port)) + wire, _ = sock.recvfrom(65535) + accept = decode_rudp_packet(wire) + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + session.receive(accept, 0) + return sock, session + + def test_the_goodbye_for_a_refused_reliable_frame_does_not_acknowledge_it(self) -> None: + import dataclasses + + from cultnet_py import CultNetRudpSendOptions, encode_rudp_packet + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + sock, session = self._raw_peer(server) + # A reliable fragment whose metadata the session refuses: fragment id 0 names no set. + # The session records its sequence before it refuses it. + (fragment,) = session.send_many("schema", b"x", CultNetRudpSendOptions(reliable=True, ordered=True)) + refused = dataclasses.replace(fragment, fragment_id=0, fragment_index=0, fragment_count=2) + sock.sendto(encode_rudp_packet(refused), ("127.0.0.1", server.port)) + goodbye = None + for _ in range(40): + try: + wire, _ = sock.recvfrom(65535) + except TimeoutError: + continue + packet = decode_rudp_packet(wire) + if packet.packet_type == CultNetRudpPacketType.DISCONNECT: + goodbye = packet + break + self.assertIsNotNone(goodbye) + self.assertEqual(goodbye.payload, b"session refused a packet") + distance = goodbye.ack - refused.sequence + acknowledged = distance == 0 or (0 < distance <= 32 and goodbye.ack_mask & (1 << (distance - 1)) != 0) + self.assertFalse(acknowledged, "the goodbye acknowledged the refused frame") + finally: + server.stop() + + def test_responses_after_one_that_can_never_be_sent_are_not_sent(self) -> None: + from cultnet_py import CultNetRudpSendOptions, encode_rudp_packet + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + sock, session = self._raw_peer(server) + addr = sock.getsockname()[:2] + request = hello(runtime_id="probe").to_bytes() + options = CultNetRudpSendOptions(reliable=True, ordered=True) + first = session.send("schema", request, options) + second = session.send("schema", request, options) + attempts: list[CultNetRudpPacketType] = [] + failures = [1] + + def sendto(sock_: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(addr): + packet_type = decode_rudp_packet(data).packet_type + attempts.append(packet_type) + if packet_type == CultNetRudpPacketType.DATA and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock_, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # The second frame waits for the first, so one receive delivers both, and + # the first of the two responses can never be sent as built. + sock.sendto(encode_rudp_packet(second), ("127.0.0.1", server.port)) + time.sleep(0.1) + attempts.clear() + sock.sendto(encode_rudp_packet(first), ("127.0.0.1", server.port)) + self._wait_for(lambda: CultNetRudpPacketType.DISCONNECT in attempts) + time.sleep(0.1) + + # The unsendable response, then only the goodbye: no second response, no ack. + self.assertEqual(attempts, [CultNetRudpPacketType.DATA, CultNetRudpPacketType.DISCONNECT]) + self.assertTrue(server._rudp_thread.is_alive()) + finally: + server.stop() + + def test_a_peer_ended_in_the_resend_loop_is_sent_nothing_more_from_it(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + mode = ["transient"] + attempts: list[CultNetRudpPacketType] = [] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr): + packet_type = decode_rudp_packet(data).packet_type + if mode[0] != "transient": + attempts.append(packet_type) + # Only responses fail; X's requests are acknowledged, so X goes quiet. + if packet_type == CultNetRudpPacketType.DATA and mode[0] == "transient": + raise OSError(errno.ENOBUFS, "injected transient failure") + if packet_type == CultNetRudpPacketType.DATA and mode[0] == "permanent": + mode[0] = "healthy" + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + x.send("schema", hello(runtime_id="one").to_bytes()) + x.send("schema", hello(runtime_id="two").to_bytes()) + for _ in range(40): + x.receive_once() + time.sleep(0.005) + self._wait_for(lambda: server.rudp_send_failures >= 2) + # Both responses are owed; the next resend of the first can never be sent. + mode[0] = "permanent" + self._wait_for(lambda: CultNetRudpPacketType.DISCONNECT in attempts) + time.sleep(0.2) + + self.assertEqual(attempts, [CultNetRudpPacketType.DATA, CultNetRudpPacketType.DISCONNECT]) + self.assertTrue(server._rudp_thread.is_alive()) + finally: + server.stop() + + def _wait_for(self, predicate: Callable[[], bool]) -> None: + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.005) + self.fail("condition not reached") + + if __name__ == "__main__": unittest.main() diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 47348bfe..c0cd200f 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -3,9 +3,11 @@ use cultnet_rs::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRudpPacket, CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, CultNetWireContract, decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, - encode_rudp_packet, + encode_rudp_packet, is_permanent_send_error, }; use std::collections::BTreeMap; +#[cfg(test)] +use std::collections::BTreeSet; use std::io::ErrorKind; use std::net::{SocketAddr, UdpSocket}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -208,6 +210,14 @@ pub struct CultMeshRudpDocumentServer { clock: C, options: CultMeshRudpDocumentServerOptions, packets_dropped: u64, + send_failures: u64, + /// Peers whose every datagram fails to send, standing in for an unroutable + /// or full path that a loopback peer cannot be made to have. + #[cfg(test)] + failing_peers: BTreeSet, + /// Peers whose next datagram after this many fails permanently, once. + #[cfg(test)] + unsendable_after: BTreeMap, } impl CultMeshRudpDocumentServer @@ -236,6 +246,11 @@ where clock, options, packets_dropped: 0, + send_failures: 0, + #[cfg(test)] + failing_peers: BTreeSet::new(), + #[cfg(test)] + unsendable_after: BTreeMap::new(), }) } @@ -247,6 +262,18 @@ where self.packets_dropped } + /// Datagrams that could not be sent to a peer and may yet be: no route, full + /// buffers, a refused path. Each is that peer's lost datagram: a reliable + /// packet stays pending and is resent, and the peer's session ends by idle + /// timeout, lifetime, refusal or Disconnect, never by the failure. Never an + /// error: one unreachable peer must not stop the poll that serves the + /// others. A datagram that can never be sent as built + /// (`is_permanent_send_error`) is not counted here: it ends that peer's + /// session, and the peer's goodbye names the error. + pub fn send_failures(&self) -> u64 { + self.send_failures + } + pub fn local_addr(&self) -> Result { Ok(self.socket.local_addr()?) } @@ -265,7 +292,14 @@ where let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES]; let (received, remote_addr) = match self.socket.recv_from(&mut wire) { Ok(value) => value, - Err(error) if error.kind() == ErrorKind::WouldBlock => { + // Windows reports an earlier ICMP port-unreachable on the next + // receive; it names no datagram of this poll. + Err(error) + if matches!( + error.kind(), + ErrorKind::WouldBlock | ErrorKind::ConnectionReset + ) => + { return Ok(CultMeshRudpPollOutcome::Idle); } Err(error) => return Err(error.into()), @@ -331,9 +365,10 @@ where } }; - if let Some(reply) = result.reply { - self.send_packet(key.remote_addr, &reply)?; - } + let mut unsendable = match result.reply { + Some(reply) => self.send_packet(key.remote_addr, &reply)?, + None => None, + }; if result.disconnected { self.sessions.remove(&key); @@ -363,14 +398,17 @@ where } } - if packet.reliable { + if packet.reliable && unsendable.is_none() { let ack = self .sessions .get_mut(&key) .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before ACK"))? .session .create_ack(); - self.send_packet(key.remote_addr, &ack)?; + unsendable = self.send_packet(key.remote_addr, &ack)?; + } + if let Some(error) = unsendable { + self.end_unsendable_session(key, &error)?; } Ok(CultMeshRudpPollOutcome::Handled) } @@ -390,11 +428,17 @@ where let mut resends = Vec::new(); for (key, entry) in &mut self.sessions { for packet in entry.session.due_resends(now) { - resends.push((key.remote_addr, packet)); + resends.push((*key, packet)); } } - for (remote_addr, packet) in &resends { - self.send_packet(*remote_addr, packet)?; + for (key, packet) in &resends { + // A session ended earlier in this loop has nothing left to resend. + if !self.sessions.contains_key(key) { + continue; + } + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + self.end_unsendable_session(*key, &error)?; + } } Ok(CultMeshRudpMaintenance { sessions_expired: expired, @@ -456,7 +500,10 @@ where accept } }; - self.send_packet(key.remote_addr, &reply)?; + if let Some(error) = self.send_packet(key.remote_addr, &reply)? { + self.end_unsendable_session(key, &error)?; + return Ok(false); + } Ok(true) } @@ -592,7 +639,14 @@ where entry.admitted_payload_bytes = entry.admitted_payload_bytes.saturating_add(payload_bytes); for packet in &packets { - self.send_packet(key.remote_addr, packet)?; + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason: format!("snapshot response could not be sent: {error}"), + })); + } } } _ => {} @@ -607,7 +661,23 @@ where return Ok(()); }; let goodbye = entry.session.end_refused_session(); - self.send_packet(key.remote_addr, &goodbye) + self.send_packet(key.remote_addr, &goodbye)?; + Ok(()) + } + + /// The session owes its peer a packet that can never be sent as built. The + /// session ends, never the poll, and the goodbye names the error. + fn end_unsendable_session( + &mut self, + key: CultMeshRudpSessionKey, + error: &std::io::Error, + ) -> Result<()> { + let Some(mut entry) = self.sessions.remove(&key) else { + return Ok(()); + }; + let goodbye = entry.session.end_unsendable_session(error); + self.send_packet(key.remote_addr, &goodbye)?; + Ok(()) } fn payload_budget_allows(&self, key: CultMeshRudpSessionKey, bytes: usize) -> bool { @@ -633,13 +703,48 @@ where .is_some_and(|total| total <= self.options.max_admitted_payload_bytes) } - fn send_packet(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + /// A transient send failure is that peer's lost datagram: counted, never an + /// error. A permanent one (`is_permanent_send_error`) is returned as `Some` + /// for the caller to end the session over. Only an encode failure, the + /// server's own bug, is an error. + fn send_packet( + &mut self, + remote_addr: SocketAddr, + packet: &CultNetRudpPacket, + ) -> Result> { let wire = encode_rudp_packet(packet)?; - self.socket.send_to(&wire, remote_addr)?; - Ok(()) + match self.send_datagram(&wire, remote_addr) { + Ok(_) => {} + Err(error) if is_permanent_send_error(&error) => return Ok(Some(error)), + Err(_) => self.send_failures += 1, + } + Ok(None) + } + + fn send_datagram(&mut self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { + #[cfg(test)] + { + match self.unsendable_after.get(&remote_addr).copied() { + Some(0) => { + self.unsendable_after.remove(&remote_addr); + return Err(std::io::Error::from(ErrorKind::InvalidInput)); + } + Some(remaining) => { + self.unsendable_after.insert(remote_addr, remaining - 1); + } + None => {} + } + if self.failing_peers.contains(&remote_addr) { + return Err(std::io::Error::other("injected send failure")); + } + } + self.socket.send_to(wire, remote_addr) } } +#[cfg(test)] +mod send_failure_tests; + fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { if options.max_sessions == 0 { return Err(anyhow!("max_sessions must be greater than zero")); diff --git a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs new file mode 100644 index 00000000..68351b35 --- /dev/null +++ b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs @@ -0,0 +1,446 @@ +//! One peer whose datagrams cannot be sent must not stop the server serving +//! the others. The failure is injected in `send_packet` through +//! `failing_peers`; a loopback peer cannot be made unroutable on demand. + +use super::*; +use cultnet_rs::{ + CultNetRawPayloadEncoding, CultNetRudpSocketMode, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, +}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; + +#[derive(Clone)] +struct Clock(Arc); + +impl CultMeshRudpServerClock for Clock { + fn now_unix_millis(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } + fn now_monotonic_millis(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } +} + +type Received = Arc>>; +type Server = CultMeshRudpDocumentServer< + Box Result<()>>, + Box Result>>, + Clock, +>; +type Client = CultNetRudpSocketTransportConnection; + +fn document(key: &str) -> CultNetRawDocumentRecord { + CultNetRawDocumentRecord { + schema_id: "test.raw.v1".into(), + record_key: key.into(), + stored_at: "2026-09-03T00:00:00Z".into(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: vec![0x81, 0xa1, 0x78, 0x2a], + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + } +} + +fn client(target: SocketAddr, id: u32) -> (Client, SocketAddr) { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.set_nonblocking(true).unwrap(); + let addr = socket.local_addr().unwrap(); + let client = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, + runtime_id: "test-client".into(), + socket, + mode: CultNetRudpSocketMode::Client, + remote_addr: Some(target), + connection_id: id, + initial_sequence: Some(1), + resend_delay_ms: 10, + transport_id: None, + max_payload_bytes: None, + max_fragment_bytes: Some(1200), + max_pending_reliable_packets: Some(64), + media_reliable_expire_after_ms: None, + reconnect_policy: None, + }) + .unwrap(); + (client, addr) +} + +fn send(client: &mut Client, message: &CultNetMessage) { + client + .send( + "schema", + encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap(), + ) + .unwrap(); +} + +/// A server with a 1 s idle timeout and two connected peers, the first +/// returned being the one whose address sorts first. +fn two_peers() -> ( + Server, + Clock, + Received, + (Client, SocketAddr), + (Client, SocketAddr), +) { + let clock = Clock(Arc::new(AtomicU64::new(1_000))); + let received: Received = Arc::default(); + let sink_received = received.clone(); + let sink: Box Result<()>> = + Box::new(move |receipt| { + sink_received.lock().unwrap().push(receipt.message_id); + Ok(()) + }); + let source: Box< + dyn FnMut(&CultMeshRudpSnapshotQuery) -> Result>, + > = Box::new(|_| Ok(vec![document("catalog")])); + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0").unwrap(), + sink, + source, + clock.clone(), + CultMeshRudpDocumentServerOptions { + session_idle_timeout: Duration::from_secs(1), + resend_delay: Duration::from_millis(10), + ..Default::default() + }, + ) + .unwrap(); + let target = server.local_addr().unwrap(); + let (mut a, a_addr) = client(target, 7); + let (mut b, b_addr) = client(target, 7); + a.connect(Vec::new()).unwrap(); + b.connect(Vec::new()).unwrap(); + for _ in 0..500 { + server.poll_once().unwrap(); + a.receive_once().unwrap(); + b.receive_once().unwrap(); + if a.connected() && b.connected() { + break; + } + thread::sleep(Duration::from_millis(1)); + } + assert!(a.connected() && b.connected()); + for _ in 0..4 { + server.poll_once().unwrap(); + } + assert_eq!(server.session_count(), 2); + if a_addr < b_addr { + (server, clock, received, (a, a_addr), (b, b_addr)) + } else { + (server, clock, received, (b, b_addr), (a, a_addr)) + } +} + +#[test] +fn one_peer_whose_sends_fail_does_not_stop_the_server_serving_the_others() { + let (mut server, clock, received, (mut x, x_addr), (mut y, _)) = two_peers(); + server.failing_peers.insert(x_addr); + + // X's reply, ack and snapshot response all fail to send. X is visited first. + send( + &mut x, + &CultNetMessage::SnapshotRequest { + message_id: "x-snapshot".into(), + schema_ids: None, + record_keys: None, + }, + ); + send( + &mut y, + &CultNetMessage::DocumentPutRaw { + message_id: "y-put".into(), + document: document("y"), + }, + ); + send( + &mut y, + &CultNetMessage::SnapshotRequest { + message_id: "y-snapshot".into(), + schema_ids: None, + record_keys: None, + }, + ); + let mut y_response = None; + for step in 0..200 { + // The clock moves past X's resend delay so `maintain` has X's resend due. + clock.0.store(1_000 + step * 15, Ordering::SeqCst); + // Never `Err`: X's failed sends are its own lost datagrams. + server.poll_once().unwrap(); + if let Some(frame) = y.receive_once().unwrap() { + y_response = Some(frame); + break; + } + thread::sleep(Duration::from_millis(1)); + } + let frame = y_response.expect("Y receives its snapshot response"); + let response = + decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0) + .unwrap(); + assert!(matches!( + response, + CultNetMessage::SnapshotResponseRaw { ref message_id, .. } if message_id == "y-snapshot" + )); + assert!(server.send_failures() > 0); + assert_eq!(received.lock().unwrap().as_slice(), ["y-put"]); + assert!(server.maintain().is_ok()); + + // X's session ends by the idle rule that already ends sessions, not by the + // failure: Y stays active, X does not. + clock.0.store(1_900, Ordering::SeqCst); + send( + &mut y, + &CultNetMessage::DocumentPutRaw { + message_id: "y-put-2".into(), + document: document("y2"), + }, + ); + for _ in 0..20 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(server.session_count(), 2, "the failure ended no session"); + clock.0.store(2_500, Ordering::SeqCst); + let maintenance = server.maintain().unwrap(); + assert_eq!(maintenance.sessions_expired, 1); + assert_eq!(server.session_count(), 1); +} + +// A permanent send failure is one that can never succeed for the datagram as +// built (`is_permanent_send_error`). It ends that peer's session, never the +// poll, and the peer's goodbye names the error. + +fn put(id: &str) -> CultNetMessage { + CultNetMessage::DocumentPutRaw { + message_id: id.into(), + document: document(id), + } +} + +fn snapshot(id: &str) -> CultNetMessage { + CultNetMessage::SnapshotRequest { + message_id: id.into(), + schema_ids: None, + record_keys: None, + } +} + +fn goodbye_reason(client: &mut Client) -> Vec { + for _ in 0..50 { + client.receive_once().unwrap(); + if let Some(reason) = client.disconnect_reason() { + return reason.to_vec(); + } + thread::sleep(Duration::from_millis(2)); + } + panic!("the client was never told the session ended"); +} + +fn is_unsendable_reason(reason: &[u8]) -> bool { + reason.starts_with(b"packet could not be sent: ") + && reason.len() > b"packet could not be sent: ".len() +} + +#[test] +fn a_permanent_failure_acknowledging_a_document_ends_the_session_after_the_document_is_kept() { + let (mut server, _clock, received, (mut x, x_addr), (mut y, _)) = two_peers(); + server.unsendable_after.insert(x_addr, 0); + send(&mut x, &put("x-put")); + send(&mut y, &put("y-put")); + + for _ in 0..10 { + // Never `Err`: the failure ends X's session, not the poll. + server.poll_once().unwrap(); + } + let mut kept = received.lock().unwrap().clone(); + kept.sort(); + assert_eq!(kept, ["x-put", "y-put"]); + assert_eq!(server.session_count(), 1); + assert_eq!(server.send_failures(), 0); + assert!(is_unsendable_reason(&goodbye_reason(&mut x))); +} + +#[test] +fn a_permanent_failure_sending_a_snapshot_response_rejects_it_naming_the_error() { + let (mut server, _clock, _received, (mut x, x_addr), (_y, _)) = two_peers(); + server.unsendable_after.insert(x_addr, 0); + send(&mut x, &snapshot("x-snapshot")); + + let mut rejection = None; + for _ in 0..10 { + if let CultMeshRudpPollOutcome::ApplicationRejected(found) = server.poll_once().unwrap() { + rejection = Some(found); + break; + } + } + let rejection = rejection.expect("the response can never be sent"); + assert_eq!(rejection.operation, CultMeshRudpApplicationOperation::SnapshotRequest); + assert_eq!(rejection.message_id, "x-snapshot"); + assert!( + rejection + .reason + .starts_with("snapshot response could not be sent: ") + && rejection.reason.len() > "snapshot response could not be sent: ".len(), + "{}", + rejection.reason + ); + assert_eq!(server.session_count(), 1); +} + +#[test] +fn a_permanent_failure_in_resends_ends_only_that_peers_session() { + let (mut server, clock, received, (mut x, x_addr), (mut y, _)) = two_peers(); + // X's snapshot response is lost, so a resend is due; then the datagram can + // never be sent as built. + server.failing_peers.insert(x_addr); + send(&mut x, &snapshot("x-snapshot")); + for _ in 0..5 { + server.poll_once().unwrap(); + } + server.failing_peers.clear(); + server.unsendable_after.insert(x_addr, 0); + clock.0.store(1_200, Ordering::SeqCst); + + let maintenance = server.maintain().unwrap(); + assert!(maintenance.packets_resent > 0); + assert_eq!(server.session_count(), 1); + assert!(is_unsendable_reason(&goodbye_reason(&mut x))); + + send(&mut y, &put("y-put")); + for _ in 0..5 { + server.poll_once().unwrap(); + } + assert_eq!(received.lock().unwrap().as_slice(), ["y-put"]); +} + +#[test] +fn a_connect_whose_accept_can_never_be_sent_starts_no_session() { + let (mut server, _clock, _received, _x, _y) = two_peers(); + let (mut z, z_addr) = client(server.local_addr().unwrap(), 9); + server.unsendable_after.insert(z_addr, 0); + z.connect(Vec::new()).unwrap(); + + for _ in 0..10 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(server.session_count(), 2); + assert_eq!(server.packets_dropped(), 1); +} + +#[test] +fn healthy_sends_are_not_failures() { + let (mut server, _clock, _received, (mut x, _), (_y, _)) = two_peers(); + send(&mut x, &put("x-put")); + send(&mut x, &snapshot("x-snapshot")); + for _ in 0..10 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(server.send_failures(), 0); + assert_eq!(server.session_count(), 2); +} + +#[test] +fn a_session_ended_in_the_resend_loop_is_sent_nothing_more_from_it() { + let (mut server, clock, _received, (mut x, x_addr), (_y, _)) = two_peers(); + server.failing_peers.insert(x_addr); + send(&mut x, &snapshot("one")); + send(&mut x, &snapshot("two")); + for _ in 0..10 { + server.poll_once().unwrap(); + } + let before = server.send_failures(); + + server.unsendable_after.insert(x_addr, 0); + clock.0.store(1_200, Ordering::SeqCst); + let maintenance = server.maintain().unwrap(); + + // Both responses are due. The first can never be sent and ends the session; + // the goodbye is the only other datagram attempted at X. + assert_eq!(maintenance.packets_resent, 2); + assert_eq!(server.send_failures() - before, 1); + assert_eq!(server.session_count(), 1); +} + +#[test] +fn no_ack_follows_a_reply_that_can_never_be_sent() { + let (mut server, _clock, _received, _x, _y) = two_peers(); + let target = server.local_addr().unwrap(); + // A peer driven packet by packet: a client never sends a reliable Ping, but a + // peer may, and the server both answers it and acknowledges it. + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let addr = socket.local_addr().unwrap(); + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 7, + ..Default::default() + }); + let connect = session.create_connect(0, Vec::new()).unwrap(); + socket + .send_to(&encode_rudp_packet(&connect).unwrap(), target) + .unwrap(); + let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES]; + for _ in 0..10 { + server.poll_once().unwrap(); + } + let (read, _) = socket.recv_from(&mut wire).unwrap(); + let accept = decode_rudp_packet(&wire[..read]).unwrap(); + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + session.receive(&accept, 0).unwrap(); + assert_eq!(server.session_count(), 3); + + let mut ping = session.create_ping(b"p".to_vec()); + ping.reliable = true; + ping.sequence = connect.sequence.wrapping_add(1); + // The Pong can never be sent; the ack after it could. + server.unsendable_after.insert(addr, 0); + socket + .send_to(&encode_rudp_packet(&ping).unwrap(), target) + .unwrap(); + for _ in 0..10 { + server.poll_once().unwrap(); + } + + // The session ends over the Pong: the peer gets its goodbye and nothing else. + assert_eq!(server.session_count(), 2); + let mut arrived = Vec::new(); + while let Ok((read, _)) = socket.recv_from(&mut wire) { + arrived.push(decode_rudp_packet(&wire[..read]).unwrap()); + } + assert_eq!(arrived.len(), 1, "{arrived:?}"); + assert_eq!(arrived[0].packet_type, CultNetRudpPacketType::Disconnect); + assert!(is_unsendable_reason(&arrived[0].payload)); +} + +/// Windows reports an ICMP port-unreachable for an earlier datagram as +/// `ConnectionReset` on the next receive. It names no datagram of this poll, so +/// the poll is idle and the server goes on serving. Only Windows reports it, so +/// only a Windows run can see the rule break. +#[cfg(windows)] +#[test] +fn a_reset_reported_by_a_receive_is_idle_not_an_error() { + let (mut server, _clock, received, (mut x, _), (mut y, _)) = two_peers(); + // X asks for a snapshot and goes away before the answer. + send(&mut x, &snapshot("x-snapshot")); + drop(x); + for _ in 0..20 { + server + .poll_once() + .expect("a reset from X's closed port is not an error"); + thread::sleep(Duration::from_millis(1)); + } + + send(&mut y, &put("y-put")); + for _ in 0..20 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(received.lock().unwrap().as_slice(), ["y-put"]); +} diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index e9813e18..2e487c77 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -22,7 +22,9 @@ import { defineCultNetDocumentBinding, encodeCultNetMessageForWire, encodeRudpPacket, + isPermanentSendError, parseCultNetMessage, + sendRudpDatagram, type CultNetErrorMessage, type CultNetDocumentBinding, type CultNetMessage, @@ -30,6 +32,7 @@ import { type CultNetOperationResponseMessage, type CultNetRawDocumentRecord, type CultNetReconnectPolicy, + type CultNetRudpPacket, type CultNetSchemaCatalogOptions, type CultNetSnapshotResponseRawMessage, type CultNetWireContract, @@ -4307,6 +4310,12 @@ export interface CultMeshRudpServerSession { export interface CultMeshRudpDocumentServer { readonly bind: { host: string; port: number }; + /** + * Datagrams that could not be sent to a peer. Each is that peer's lost datagram: a + * reliable packet stays pending and is resent, and no other peer is affected. A datagram + * that can never be sent as built is not counted here: it ends that peer's session. + */ + readonly sendFailures: number; start(): Promise; close(): void; } @@ -5767,15 +5776,19 @@ export class CultMesh { const port = options.bindPort ?? 0; const bind = { host, port }; const socket = options.socket ?? createSocket(host.includes(":") ? "udp6" : "udp4"); - const sessions = new Map; - }>(); + }; + const sessions = new Map(); let sessionSequence = 0; + let sendFailures = 0; const resendPollMs = Math.max(10, options.resendPollMs ?? 25); const sessionTimeoutMs = Math.max(1_000, options.sessionTimeoutMs ?? 30_000); const wireContract = options.wireContract ?? "cultnet.schema.v0"; @@ -5800,19 +5813,15 @@ export class CultMesh { if (packet.packetType === "connect") { const connectPayload = Uint8Array.from(packet.payload ?? []); if (record?.session.connectRepeats(packet)) { - socket.send( - encodeRudpPacket(record.session.acceptConnect(packet, nowMs)), - remote.port, - remote.address, - ); + sendPacket(record, record.session.acceptConnect(packet, nowMs)); return; } - if (record) closeSession(record); - record = { + const next: SessionRecord = { sessionId: `${runtimeId}:${++sessionSequence}`, remote: { address: remote.address, family: remote.family, port: remote.port }, connectPayload, closed: false, + unsendable: false, session: new CultNetRudpSession({ connectionId, initialSequence: options.initialSequence, @@ -5821,8 +5830,12 @@ export class CultMesh { }), work: Promise.resolve(), }; - sessions.set(key, record); - socket.send(encodeRudpPacket(record.session.acceptConnect(packet, nowMs)), remote.port, remote.address); + sendPacket(next, next.session.acceptConnect(packet, nowMs)); + // A peer whose Accept can never be sent cannot be answered: no session + // starts, and one already there is left as it was. + if (next.unsendable) return; + if (record) closeSession(record); + sessions.set(key, next); return; } @@ -5830,12 +5843,22 @@ export class CultMesh { return; } - const result = record.session.receive(packet, nowMs); + let result: ReturnType; + try { + result = record.session.receive(packet, nowMs); + } catch { + // receive() has already recorded the packet's reliable sequence, so the + // session cannot be kept: a retransmit would be acknowledged and the frame + // silently lost. It ends, never the server, and the peer is told. What a + // datagram carries is the sender's business, so nothing is reported. + endRefusedSession(key, record); + return; + } if (result.reply) { - socket.send(encodeRudpPacket(result.reply), record.remote.port, record.remote.address); + sendPacket(record, result.reply); } for (const ready of result.readyToSend ?? []) { - socket.send(encodeRudpPacket(ready), record.remote.port, record.remote.address); + sendPacket(record, ready); } for (const frame of result.delivered) { if (frame.channelId !== "schema") { @@ -5851,11 +5874,7 @@ export class CultMesh { return; } if (packet.reliable) { - socket.send( - encodeRudpPacket(record.session.createAckForReceived(packet.sequence)), - record.remote.port, - record.remote.address, - ); + sendPacket(record, record.session.createAckForReceived(packet.sequence)); } } catch (error) { reportError(error); @@ -5864,7 +5883,7 @@ export class CultMesh { socket.on("error", reportError); async function handleDocumentServerFrame( - record: { sessionId: string; remote: { address: string; family: string; port: number }; session: CultNetRudpSession; connectPayload: Uint8Array; closed: boolean }, + record: SessionRecord, payload: Uint8Array, ): Promise { if (record.closed) return; @@ -5926,13 +5945,7 @@ export class CultMesh { } } - function publicSession(record: { - sessionId: string; - remote: { address: string; family: string; port: number }; - session: CultNetRudpSession; - connectPayload: Uint8Array; - closed: boolean; - }): CultMeshRudpServerSession { + function publicSession(record: SessionRecord): CultMeshRudpServerSession { return { sessionId: record.sessionId, remote: { ...record.remote }, @@ -5944,31 +5957,58 @@ export class CultMesh { }; } - function notifySessionClosed(record: { - sessionId: string; - remote: { address: string; family: string; port: number }; - session: CultNetRudpSession; - connectPayload: Uint8Array; - closed: boolean; - }): void { + function notifySessionClosed(record: SessionRecord): void { if (!options.onSessionClosed) return; void Promise.resolve(options.onSessionClosed(publicSession(record))).catch(reportError); } - function closeSession(record: { - sessionId: string; - remote: { address: string; family: string; port: number }; - session: CultNetRudpSession; - connectPayload: Uint8Array; - closed: boolean; - }): void { + function closeSession(record: SessionRecord): void { if (record.closed) return; record.closed = true; notifySessionClosed(record); } + /** + * Sends one packet to a session's peer. A failure is that peer's alone: one that may + * pass (no route, full buffers) is a lost datagram, counted, and resent if it was + * reliable; one that can never pass as built ends that peer's session. Neither ends + * the server, a timer or another peer's session. + */ + function sendPacket(record: SessionRecord, packet: CultNetRudpPacket): void { + sendRudpDatagram(socket, encodeRudpPacket(packet), record.remote.port, record.remote.address, (error) => { + if (isPermanentSendError(error)) { + endUnsendableSession(record, error); + } else { + sendFailures += 1; + } + }); + } + + function endRefusedSession(key: string, record: SessionRecord): void { + sessions.delete(key); + closeSession(record); + // The goodbye is built after the reset, or its ack field would acknowledge the + // very frame the session refused. + record.session.resetPeerState(); + const goodbye = encodeRudpPacket(record.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + sendRudpDatagram(socket, goodbye, record.remote.port, record.remote.address, () => {}); + } + + /** The session owes its peer a packet that can never be sent as built. */ + function endUnsendableSession(record: SessionRecord, error: Error): void { + if (record.unsendable) return; + record.unsendable = true; + const key = `${record.remote.address}:${record.remote.port}`; + // A session that never started, or was already replaced, has nothing to end. + if (sessions.get(key) !== record) return; + sessions.delete(key); + closeSession(record); + const goodbye = encodeRudpPacket(record.session.endUnsendableSession(error)); + sendRudpDatagram(socket, goodbye, record.remote.port, record.remote.address, () => {}); + } + function sendSchemaMessage( - record: { sessionId: string; remote: { address: string; port: number }; session: CultNetRudpSession; closed: boolean }, + record: SessionRecord, message: CultNetMessage, ): void { if (record.closed) throw new Error(`CultMesh RUDP session ${record.sessionId} is closed.`); @@ -5978,12 +6018,15 @@ export class CultMesh { ordered: true, nowMs: Date.now(), })) { - socket.send(encodeRudpPacket(packet), record.remote.port, record.remote.address); + sendPacket(record, packet); } } return { bind, + get sendFailures() { + return sendFailures; + }, async start(): Promise { await new Promise((resolve, reject) => { socket.once("error", reject); @@ -6005,7 +6048,7 @@ export class CultMesh { continue; } for (const packet of record.session.dueResends(nowMs)) { - socket.send(encodeRudpPacket(packet), record.remote.port, record.remote.address); + sendPacket(record, packet); } } }, resendPollMs); diff --git a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts new file mode 100644 index 00000000..64f19111 --- /dev/null +++ b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts @@ -0,0 +1,221 @@ +// A datagram the document server cannot send to one peer is that peer's problem +// alone. It never escapes a timer or the receive handler, never stops the server +// serving the others, and only a datagram that can never be sent as built ends +// the session that owes it. +import assert from "node:assert/strict"; +import { createSocket, type RemoteInfo, type Socket } from "node:dgram"; +import test from "node:test"; +import { encode } from "@msgpack/msgpack"; +import { + CultNetDocumentRegistry, + CultNetRudpSession, + decodeRudpPacket, + encodeRudpPacket, + type CultNetPeer, +} from "cultnet-ts"; +import { CultMesh } from "../src/index"; + +async function waitFor(predicate: () => boolean, description: string): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > 3_000) { + throw new Error(`Timed out waiting for ${description}.`); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function watchUncaught(): { errors: unknown[]; stop: () => void } { + const errors: unknown[] = []; + const listener = (error: unknown) => errors.push(error); + process.on("uncaughtException", listener); + return { errors, stop: () => process.off("uncaughtException", listener) }; +} + +function put(recordKey: string) { + return { + schemaVersion: "cultnet.document_put_raw.v0" as const, + messageId: `put-${recordKey}`, + document: { + schemaId: "cultmesh.note.v0", + recordKey, + storedAt: new Date().toISOString(), + payloadEncoding: "messagepack" as const, + payload: encode(recordKey), + }, + }; +} + +interface Server { + socket: Socket; + server: ReturnType; + /** The record key of every put received, with the port it came from. */ + puts: Map; + closed: number[]; + errors: Error[]; + peer(name: string): Promise; +} + +async function startServer(connectionId: number, socket: Socket): Promise { + const puts = new Map(); + const closed: number[] = []; + const errors: Error[] = []; + const server = CultMesh.createRudpDocumentServer("send-failure-server", connectionId, { + documents: new CultNetDocumentRegistry(), + socket, + bindHost: "127.0.0.1", + bindPort: 0, + resendDelayMs: 10, + resendPollMs: 10, + sessionTimeoutMs: 1_000, + onDocumentPutRaw: (document) => { + puts.set(document.recordKey, document.remote.port); + }, + onSessionClosed: (session) => { + closed.push(session.remote.port); + }, + onError: (error) => errors.push(error), + }); + await server.start(); + const peer = (name: string) => + CultMesh.createRudpPeer(name, connectionId, `rudp://127.0.0.1:${server.bind.port}`, { + resendDelayMs: 10, + resendPollMs: 5, + connectTimeoutMs: 2_000, + }); + return { socket, server, puts, closed, errors, peer }; +} + +test("a Connect whose Accept can never be sent starts no session, and the server keeps serving", async () => { + const connectionId = 0x10203091; + const uncaught = watchUncaught(); + const socket = createSocket("udp4"); + const served = await startServer(connectionId, socket); + let peer: CultNetPeer | undefined; + try { + // A Connect as it arrives from UDP source port 0, which Node refuses to answer. + const wire = Buffer.from(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(Date.now()))); + const remote: RemoteInfo = { address: "127.0.0.1", family: "IPv4", port: 0, size: wire.length }; + socket.emit("message", wire, remote); + // Past the idle timeout: a session that had started would have been closed by now. + await sleep(1_300); + + peer = await served.peer("after-port-zero"); + peer.send(put("after")); + await waitFor(() => served.puts.has("after"), "the real peer's put"); + assert.deepEqual(served.closed, [], "no session started for port 0"); + assert.deepEqual(served.errors.map((error) => error.message), []); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + peer?.close(); + served.server.close(); + } +}); + +test("a datagram that may yet pass is counted; one that never can ends only that peer's session", async () => { + const connectionId = 0x10203092; + const socket = createSocket("udp4"); + // Sends to a failing port report the failure the way Node reports a system one: + // later, through the send's callback. + const failing = new Map(); + const send = socket.send.bind(socket) as (...args: unknown[]) => void; + Object.assign(socket, { + send: (message: Uint8Array, port: number, address: string, callback?: (error: Error | null) => void) => { + const code = failing.get(port); + if (code === undefined) { + send(message, port, address, callback); + return; + } + process.nextTick(() => callback?.(Object.assign(new Error(`send failed: ${code}`), { code }))); + }, + }); + const served = await startServer(connectionId, socket); + let x: CultNetPeer | undefined; + let y: CultNetPeer | undefined; + try { + x = await served.peer("x"); + y = await served.peer("y"); + x.send(put("x-1")); + await waitFor(() => served.puts.has("x-1"), "X's first put"); + const xPort = served.puts.get("x-1")!; + + // No route: X's acknowledgements are lost datagrams, counted, never reported as errors, + // and end nothing. + assert.equal(served.server.sendFailures, 0); + failing.set(xPort, "EHOSTUNREACH"); + x.send(put("x-2")); + await waitFor(() => served.server.sendFailures >= 2, "the counted losses"); + y.send(put("y-1")); + await waitFor(() => served.puts.has("y-1"), "Y's put"); + assert.deepEqual(served.closed, [], "a loss that may pass ends no session"); + assert.deepEqual(served.errors.map((error) => error.message), [], "a lost datagram is not an error"); + + // X's retransmitted put is acknowledged again, and that datagram can never be sent. + failing.set(xPort, "EMSGSIZE"); + await waitFor(() => served.closed.length > 0, "X's session to end"); + assert.deepEqual(served.closed, [xPort]); + assert.equal( + served.errors.some((error) => (error as NodeJS.ErrnoException).code === "EMSGSIZE"), + false, + "a datagram that can never be sent ends the session instead of being reported", + ); + + y.send(put("y-2")); + await waitFor(() => served.puts.has("y-2"), "Y's put after X ended"); + assert.deepEqual(served.closed, [xPort]); + } finally { + x?.close(); + y?.close(); + served.server.close(); + } +}); + +test("a packet the session refuses ends only that peer's session, and the peer is told", async () => { + const connectionId = 0x10203093; + const socket = createSocket("udp4"); + const served = await startServer(connectionId, socket); + const raw = createSocket("udp4"); + await new Promise((resolve) => raw.bind(0, "127.0.0.1", resolve)); + const arrived: ReturnType[] = []; + raw.on("message", (wire) => arrived.push(decodeRudpPacket(wire))); + let y: CultNetPeer | undefined; + try { + const session = new CultNetRudpSession({ connectionId }); + raw.send(encodeRudpPacket(session.createConnect(Date.now())), served.server.bind.port, "127.0.0.1"); + await waitFor(() => arrived.some((packet) => packet.packetType === "accept"), "the Accept"); + session.receive(arrived.find((packet) => packet.packetType === "accept")!, Date.now()); + y = await served.peer("y"); + + // A reliable fragment whose metadata the session refuses: fragment id 0 names no set. + // The session records its sequence before it refuses it. + const [fragment] = session.sendMany("schema", new Uint8Array([1]), { reliable: true, ordered: true }); + raw.send( + encodeRudpPacket({ ...fragment!, fragmentId: 0, fragmentIndex: 0, fragmentCount: 2 }), + served.server.bind.port, + "127.0.0.1", + ); + await waitFor(() => arrived.some((packet) => packet.packetType === "disconnect"), "the goodbye"); + const goodbye = arrived.find((packet) => packet.packetType === "disconnect")!; + assert.equal(Buffer.from(goodbye.payload ?? []).toString("utf8"), "session refused a packet"); + // The goodbye's ack field does not acknowledge the refused frame, or the peer would + // count as delivered a frame that was never handled. + const acknowledged = + goodbye.ack === fragment!.sequence || + (goodbye.ack > fragment!.sequence && + goodbye.ack - fragment!.sequence <= 32 && + (goodbye.ackMask & (1 << (goodbye.ack - fragment!.sequence - 1))) !== 0); + assert.equal(acknowledged, false, "the goodbye acknowledged the refused frame"); + assert.deepEqual(served.closed, [(raw.address() as { port: number }).port]); + + y.send(put("y-after")); + await waitFor(() => served.puts.has("y-after"), "Y's put"); + assert.deepEqual(served.errors.map((error) => error.message), []); + } finally { + y?.close(); + raw.close(); + served.server.close(); + } +}); diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index 1e509bda..3344f438 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -123,6 +123,7 @@ from .transport import ( TcpFramedTransportConnection, compute_reconnect_delay_ms, create_reconnect_policy, + is_permanent_send_error, create_rudp_transport_profile, create_tcp_framed_transport_profile, decode_rudp_packet, @@ -209,6 +210,7 @@ __all__ = [ "CultNetRudpReceiveResult", "CultNetRudpSendOptions", "CultNetRudpSession", + "is_permanent_send_error", "CultNetRudpSessionOptions", "CultNetRudpSocketMode", "CultNetRudpReconnectLoop", diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 1e718aab..78ae9542 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import secrets import socket import threading @@ -435,6 +436,32 @@ def create_rudp_transport_profile( } +UNSENDABLE_PACKET_REASON = "packet could not be sent" + +_PERMANENT_SEND_ERRNOS = frozenset( + code + for code in ( + errno.EMSGSIZE, + errno.EINVAL, + errno.EAFNOSUPPORT, + getattr(errno, "WSAEMSGSIZE", None), + getattr(errno, "WSAEINVAL", None), + getattr(errno, "WSAEAFNOSUPPORT", None), + ) + if code is not None +) + + +def is_permanent_send_error(error: OSError) -> bool: + """Whether a failed `sendto` can never succeed for the datagram as built, so resending + it is pointless: it is too large for a datagram (EMSGSIZE), its address is malformed for + the socket (EINVAL), or belongs to another address family (EAFNOSUPPORT), with their + Winsock equivalents. Everything else (no route, full buffers, firewall drops, a refusal + reported for an earlier datagram, an interrupted call) is a property of the path or the + moment and may pass, so it is a lost datagram, not a reason to end a session.""" + return error.errno in _PERMANENT_SEND_ERRNOS + + class CultNetRudpSession: RELIABLE_SEND_WINDOW_PACKETS = 32 RECEIVED_SEQUENCE_WINDOW = 4_096 @@ -841,6 +868,14 @@ class CultNetRudpSession: def create_ping(self, payload: bytes = b"") -> CultNetRudpPacket: return self._create_packet(CultNetRudpPacketType.PING, "control", payload) + def end_unsendable_session(self, error: OSError) -> CultNetRudpPacket: + """Ends a session that owes its peer a packet that can never be sent as built + (see `is_permanent_send_error`). Resending it would fail forever, and dropping it + would leave a reliable sequence the peer waits on for good, so the session cannot + be kept. Returns the goodbye for the peer; its reason names the error.""" + self.reset_peer_state() + return self.create_disconnect(f"{UNSENDABLE_PACKET_REASON}: {error}".encode("utf-8")) + def create_disconnect(self, reason: bytes = b"") -> CultNetRudpPacket: self._end_session() return self._create_packet(CultNetRudpPacketType.DISCONNECT, "control", reason) diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index b91493b4..765f5c37 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -879,6 +879,28 @@ class CultNetTests(unittest.TestCase): self.assertEqual(old_ack.ack, oldest_sequence) self.assertEqual(old_ack.ack_mask, 0) + def test_cultnet_rudp_only_a_datagram_that_can_never_be_sent_is_a_permanent_failure(self) -> None: + import errno + + from cultnet_py import is_permanent_send_error + + for code in (errno.EMSGSIZE, errno.EINVAL, errno.EAFNOSUPPORT): + self.assertTrue(is_permanent_send_error(OSError(code, "permanent")), errno.errorcode[code]) + for code in ( + errno.EPERM, + errno.EINTR, + errno.EAGAIN, + errno.EACCES, + errno.ENETDOWN, + errno.ENETUNREACH, + errno.EADDRNOTAVAIL, + errno.ENOBUFS, + errno.ECONNREFUSED, + errno.EHOSTUNREACH, + ): + self.assertFalse(is_permanent_send_error(OSError(code, "transient")), errno.errorcode[code]) + self.assertFalse(is_permanent_send_error(OSError("no errno"))) + # The sender's flow window: a reliable packet goes on the wire only while its sequence is at most 1,023 above # the lowest unacked one and the payload above that sequence stays within 4 MiB. FLOW_CONNECTION_ID = 0x464C4F57 diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 9e8c4a1c..f91a3f1e 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -28,6 +28,9 @@ use crate::create_reconnect_policy; use crate::decode_cultnet_message_from_slice; use crate::encode_cultnet_message_to_vec; +#[cfg(test)] +mod send_failure_tests; + const RUDP_MAGIC: [u8; 4] = [0x43, 0x4e, 0x52, 0x30]; const RUDP_VERSION: u8 = 0; pub(crate) const RUDP_FIXED_HEADER_BYTES: usize = 36; @@ -465,6 +468,28 @@ impl CultNetRudpSession { self.create_disconnect(RUDP_REFUSED_PACKET_REASON.to_vec()) } + /// Ends a session that owes its peer a packet that can never be sent as + /// built (see `is_permanent_send_error`). Resending it would fail forever, + /// and dropping it would leave a reliable sequence the peer waits on for + /// good, so the session cannot be kept. Returns the goodbye for the peer; + /// its reason names the error. + pub fn end_unsendable_session(&mut self, error: &std::io::Error) -> CultNetRudpPacket { + self.reset_peer_state(); + self.create_disconnect(format!("{RUDP_UNSENDABLE_PACKET_REASON}: {error}").into_bytes()) + } + + /// Undoes a send that never reached the wire: every packet it created at + /// or after `first_sequence`, pending or queued, leaves no trace and its + /// sequences are reissued, so the peer never sees a gap. Only valid while + /// nothing else has been sent since. + fn withdraw_send(&mut self, first_sequence: u32) { + self.pending_reliable + .retain(|sequence, _| *sequence < first_sequence); + self.queued_reliable + .retain(|(packet, _)| packet.sequence < first_sequence); + self.next_sequence = first_sequence; + } + pub fn create_connect(&mut self, now_ms: u64, payload: Vec) -> Result { // A session that has had a peer starts a new generation: nothing it // learned from that peer describes the one this Connect reaches, and @@ -1621,6 +1646,13 @@ pub struct CultNetRudpServerHub { pending_events: VecDeque, pub profile: CultNetTransportProfile, stats: CultNetTransportStats, + /// Peers whose every datagram fails to send, standing in for an unroutable + /// or full path that a loopback peer cannot be made to have. + #[cfg(test)] + failing_peers: BTreeSet, + /// Peers whose next datagram after this many fails permanently, once. + #[cfg(test)] + unsendable_after: BTreeMap, } impl CultNetRudpServerHub { @@ -1669,6 +1701,10 @@ impl CultNetRudpServerHub { pending_events: VecDeque::new(), profile, stats: CultNetTransportStats::default(), + #[cfg(test)] + failing_peers: BTreeSet::new(), + #[cfg(test)] + unsendable_after: BTreeMap::new(), }) } @@ -1709,7 +1745,7 @@ impl CultNetRudpServerHub { payload: Vec, ) -> Result<()> { let remote_addr = session.remote_addr; - let packets = { + let (first_sequence, packets) = { let peer = self .peers .get_mut(&remote_addr) @@ -1720,7 +1756,8 @@ impl CultNetRudpServerHub { session.session_generation )); } - peer.session.send_many( + let first_sequence = peer.session.next_sequence; + let packets = peer.session.send_many( channel_id, payload, channel_send_options( @@ -1730,10 +1767,25 @@ impl CultNetRudpServerHub { channel_delivery(&self.profile, channel_id), ), self.max_fragment_bytes, - )? + )?; + (first_sequence, packets) }; - for packet in packets { - self.send_packet(remote_addr, &packet)?; + for (index, packet) in packets.iter().enumerate() { + let Some(error) = self.send_packet(remote_addr, packet)? else { + continue; + }; + // A caller-directed send that can never succeed is the caller's + // error and queues nothing. Only when earlier fragments are already + // on the wire has the peer seen part of it, and then the session + // cannot be kept. + if index == 0 { + if let Some(peer) = self.peers.get_mut(&remote_addr) { + peer.session.withdraw_send(first_sequence); + } + } else { + self.end_unsendable_peer(remote_addr, &error)?; + } + return Err(error.into()); } self.stats.frames_sent += 1; Ok(()) @@ -1764,7 +1816,9 @@ impl CultNetRudpServerHub { .remove(&session.remote_addr) .expect("validated RUDP peer exists"); let packet = peer.session.create_disconnect(reason); - self.send_packet(session.remote_addr, &packet)?; + if let Some(error) = self.send_packet(session.remote_addr, &packet)? { + return Err(error.into()); + } Ok(true) } @@ -1822,7 +1876,7 @@ impl CultNetRudpServerHub { self.end_refusing_peer(remote_addr)?; return Ok(true); }; - self.send_packet(remote_addr, &reply)?; + self.send_in_poll(remote_addr, &reply)?; return Ok(true); } if !self.peers.contains_key(&remote_addr) && self.peers.len() >= self.max_peers { @@ -1851,7 +1905,11 @@ impl CultNetRudpServerHub { self.stats.packets_dropped += 1; return Ok(true); }; - self.send_packet(remote_addr, &accept)?; + if self.send_packet(remote_addr, &accept)?.is_some() { + // The peer cannot be answered, so no session starts. + self.stats.packets_dropped += 1; + return Ok(true); + } let replaced = self.peers.insert( remote_addr, CultNetRudpServerPeer { @@ -1885,14 +1943,17 @@ impl CultNetRudpServerHub { } else { None }; - if let Some(reply) = result.reply { - self.send_packet(remote_addr, &reply)?; - } - for ready in result.ready_to_send { - self.send_packet(remote_addr, &ready)?; - } - if let Some(ack) = ack { - self.send_packet(remote_addr, &ack)?; + let mut unsendable = None; + for packet in result + .reply + .iter() + .chain(&result.ready_to_send) + .chain(ack.iter()) + { + unsendable = self.send_packet(remote_addr, packet)?; + if unsendable.is_some() { + break; + } } if result.pong { self.pending_events.push_back(CultNetRudpServerEvent::Pong { @@ -1918,6 +1979,8 @@ impl CultNetRudpServerHub { reason: result.disconnect_reason, }); self.peers.remove(&remote_addr); + } else if let Some(error) = unsendable { + self.end_unsendable_peer(remote_addr, &error)?; } Ok(true) } @@ -1937,7 +2000,37 @@ impl CultNetRudpServerHub { session: peer.context, reason: goodbye.payload.clone(), }); - self.send_packet(remote_addr, &goodbye) + self.send_packet(remote_addr, &goodbye)?; + Ok(()) + } + + /// An admitted peer owes a packet that can never be sent as built. Inside a + /// poll that ends the peer's session, never the poll: the peer is told, and + /// the caller sees `Disconnected` with a reason that names the error. + fn end_unsendable_peer( + &mut self, + remote_addr: SocketAddr, + error: &std::io::Error, + ) -> Result<()> { + let Some(mut peer) = self.peers.remove(&remote_addr) else { + return Ok(()); + }; + let goodbye = peer.session.end_unsendable_session(error); + self.pending_events + .push_back(CultNetRudpServerEvent::Disconnected { + session: peer.context, + reason: goodbye.payload.clone(), + }); + self.send_packet(remote_addr, &goodbye)?; + Ok(()) + } + + /// A send inside a poll: a permanent failure ends that peer's session. + fn send_in_poll(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + if let Some(error) = self.send_packet(remote_addr, packet)? { + self.end_unsendable_peer(remote_addr, &error)?; + } + Ok(()) } pub fn poll_resends(&mut self) -> Result<()> { @@ -1952,7 +2045,10 @@ impl CultNetRudpServerHub { ); } for (remote_addr, packet) in packets { - self.send_packet(remote_addr, &packet)?; + // A peer ended earlier in this loop has nothing left to resend. + if self.peers.contains_key(&remote_addr) { + self.send_in_poll(remote_addr, &packet)?; + } } Ok(()) } @@ -1977,11 +2073,44 @@ impl CultNetRudpServerHub { timed_out.into_iter().map(|(_, context)| context).collect() } - fn send_packet(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + /// A transient send failure is that peer's lost datagram: counted, never an + /// error. A reliable packet stays pending and is resent; an unreliable one + /// was allowed to be lost. A permanent failure (`is_permanent_send_error`) + /// is returned as `Some`: the datagram can never be sent as built, and the + /// caller decides between refusing the send and ending the peer's session. + /// Only an encode failure, the hub's own bug, is an error. + fn send_packet( + &mut self, + remote_addr: SocketAddr, + packet: &CultNetRudpPacket, + ) -> Result> { let wire = encode_rudp_packet(packet)?; - let sent = self.socket.send_to(&wire, remote_addr)?; - self.stats.bytes_sent += sent as u64; - Ok(()) + match self.send_datagram(&wire, remote_addr) { + Ok(sent) => self.stats.bytes_sent += sent as u64, + Err(error) if is_permanent_send_error(&error) => return Ok(Some(error)), + Err(_) => self.stats.send_failures += 1, + } + Ok(None) + } + + fn send_datagram(&mut self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { + #[cfg(test)] + { + match self.unsendable_after.get(&remote_addr).copied() { + Some(0) => { + self.unsendable_after.remove(&remote_addr); + return Err(std::io::Error::from(ErrorKind::InvalidInput)); + } + Some(remaining) => { + self.unsendable_after.insert(remote_addr, remaining - 1); + } + None => {} + } + if self.failing_peers.contains(&remote_addr) { + return Err(std::io::Error::other("injected send failure")); + } + } + self.socket.send_to(wire, remote_addr) } } @@ -2469,6 +2598,50 @@ where /// the local caller as the disconnect reason. const RUDP_REFUSED_PACKET_REASON: &[u8] = b"session refused a packet"; const SESSION_TIMED_OUT_REASON: &[u8] = b"session timed out"; +/// Prefix of the goodbye reason for a session that owed a packet which can +/// never be sent; the error's own description follows it. +const RUDP_UNSENDABLE_PACKET_REASON: &str = "packet could not be sent"; + +/// Whether a failed `send_to` can never succeed for the datagram as built, so +/// resending it is pointless: it is too large for a datagram (`EMSGSIZE`), its +/// address is malformed for the socket (`EINVAL`), or belongs to another +/// address family (`EAFNOSUPPORT`). Everything else (no route, full buffers, +/// firewall drops, a refusal reported for an earlier datagram, an interrupted +/// call) is a property of the path or the moment and may pass, so it is a lost +/// datagram, not a reason to end a session. +pub fn is_permanent_send_error(error: &std::io::Error) -> bool { + // `EMSGSIZE` and `EAFNOSUPPORT`, or their Winsock equivalents. + #[cfg(any(target_os = "linux", target_os = "android"))] + const PERMANENT_OS_ERRORS: [i32; 2] = [90, 97]; + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + ))] + const PERMANENT_OS_ERRORS: [i32; 2] = [40, 47]; + #[cfg(windows)] + const PERMANENT_OS_ERRORS: [i32; 2] = [10040, 10047]; + #[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly", + windows + )))] + const PERMANENT_OS_ERRORS: [i32; 0] = []; + // `InvalidInput` is `EINVAL` and `WSAEINVAL`. + error.kind() == ErrorKind::InvalidInput + || error + .raw_os_error() + .is_some_and(|code| PERMANENT_OS_ERRORS.contains(&code)) +} /// A session cannot admit a Connect when its sequence space starts exhausted or /// its reliable queue holds nothing, so neither is a usable configuration. diff --git a/packages/cultnet-rs/src/rudp/send_failure_tests.rs b/packages/cultnet-rs/src/rudp/send_failure_tests.rs new file mode 100644 index 00000000..4bff8cf9 --- /dev/null +++ b/packages/cultnet-rs/src/rudp/send_failure_tests.rs @@ -0,0 +1,502 @@ +//! One peer whose datagrams cannot be sent must not stop the hub serving the +//! others. The failure is injected at `send_datagram` through `failing_peers`. + +use super::*; +use std::time::Duration; + +const CONNECTION_ID: u32 = 0x4355_4c54; + +fn socket() -> UdpSocket { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + socket +} + +fn client(remote: SocketAddr) -> CultNetRudpSocketTransportConnection { + let mut options = + CultNetRudpSocketTransportOptions::client("peer", socket(), remote, CONNECTION_ID); + options.resend_delay_ms = 10; + CultNetRudpSocketTransportConnection::new(options).unwrap() +} + +fn connect( + hub: &mut CultNetRudpServerHub, + client: &mut CultNetRudpSocketTransportConnection, +) -> CultNetRudpServerSessionContext { + client.connect(Vec::new()).unwrap(); + let mut session = None; + for _ in 0..20 { + if let Some(CultNetRudpServerEvent::Connected { session: found }) = + hub.receive_event_once().unwrap() + { + session = Some(found); + break; + } + } + client.receive_once().unwrap(); + assert!(client.connected()); + session.expect("hub saw the Connect") +} + +/// Two connected peers, ordered so the first is the one the hub visits first +/// (the peer map is ordered by address). +fn two_peers() -> ( + CultNetRudpServerHub, + ( + CultNetRudpSocketTransportConnection, + CultNetRudpServerSessionContext, + ), + ( + CultNetRudpSocketTransportConnection, + CultNetRudpServerSessionContext, + ), +) { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let mut a = client(server_addr); + let mut b = client(server_addr); + let a_session = connect(&mut hub, &mut a); + let b_session = connect(&mut hub, &mut b); + if a_session.remote_addr < b_session.remote_addr { + (hub, (a, a_session), (b, b_session)) + } else { + (hub, (b, b_session), (a, a_session)) + } +} + +#[test] +fn a_failed_resend_to_one_peer_does_not_stop_the_others_resends() { + let (mut hub, (_x, x_session), (mut y, y_session)) = two_peers(); + // Both peers' first datagram is lost, so both have a reliable resend due. + hub.failing_peers.insert(x_session.remote_addr); + hub.failing_peers.insert(y_session.remote_addr); + hub.send(&x_session, "schema", b"for-x".to_vec()).unwrap(); + hub.send(&y_session, "schema", b"for-y".to_vec()).unwrap(); + assert_eq!(hub.stats().send_failures, 2); + + // Y's path recovers; X's stays dead. X is visited first. + hub.failing_peers.remove(&y_session.remote_addr); + std::thread::sleep(Duration::from_millis(40)); + hub.poll_resends().unwrap(); + + let mut delivered = None; + for _ in 0..20 { + if let Some(frame) = y.receive_once().unwrap() { + delivered = Some(frame); + break; + } + } + assert_eq!(delivered.expect("Y receives its resend").payload, b"for-y"); + assert!(hub.stats().send_failures >= 3); + assert_eq!(hub.sessions().len(), 2, "a send failure ends no session"); +} + +#[test] +fn a_failed_reply_to_one_peer_does_not_stop_the_hub_serving_the_others() { + let (mut hub, (mut x, x_session), (mut y, _y_session)) = two_peers(); + hub.failing_peers.insert(x_session.remote_addr); + + x.send("schema", b"from-x".to_vec()).unwrap(); + y.send("schema", b"from-y".to_vec()).unwrap(); + + let mut frames = Vec::new(); + for _ in 0..20 { + if let Some(CultNetRudpServerEvent::Frame { frame, .. }) = hub.receive_event_once().unwrap() + { + frames.push(frame.payload); + } + if frames.len() == 2 { + break; + } + } + frames.sort(); + assert_eq!(frames, vec![b"from-x".to_vec(), b"from-y".to_vec()]); + assert!( + hub.stats().send_failures >= 1, + "X's ack was counted as lost" + ); + assert_eq!(hub.sessions().len(), 2); +} + +// A permanent send failure is one that can never succeed for the datagram as +// built. A caller-directed send returns it and queues nothing; inside a poll it +// ends that peer's session and never the poll. + +fn is_unsendable_reason(reason: &[u8]) -> bool { + reason.starts_with(b"packet could not be sent: ") + && reason.len() > b"packet could not be sent: ".len() +} + +fn next_event(hub: &mut CultNetRudpServerHub) -> CultNetRudpServerEvent { + for _ in 0..20 { + if let Some(event) = hub.receive_event_once().unwrap() { + return event; + } + } + panic!("the hub raised no event"); +} + +fn one_peer( + max_fragment_bytes: Option, +) -> ( + CultNetRudpServerHub, + CultNetRudpSocketTransportConnection, + CultNetRudpServerSessionContext, +) { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + options.max_fragment_bytes = max_fragment_bytes; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let mut peer = client(server_addr); + let session = connect(&mut hub, &mut peer); + (hub, peer, session) +} + +#[test] +fn an_oversized_send_returns_the_error_and_queues_nothing() { + let (mut hub, mut peer, session) = one_peer(None); + // The hub's own Accept is still awaiting its ack; the refused send adds nothing to it. + let outstanding = hub.peers[&session.remote_addr] + .session + .outstanding_reliable_packet_count(); + + let error = hub + .send(&session, "schema", vec![7; 70_000]) + .expect_err("70,000 bytes cannot be one datagram"); + let io_error = error + .downcast_ref::() + .expect("the send error is the socket's own"); + assert!(is_permanent_send_error(io_error), "{io_error}"); + assert_eq!(hub.stats().send_failures, 0); + assert_eq!(hub.stats().frames_sent, 0); + assert_eq!(hub.sessions().len(), 1, "the caller's error ends no session"); + let remote = session.remote_addr; + assert_eq!( + hub.peers[&remote].session.outstanding_reliable_packet_count(), + outstanding, + "nothing is pending or queued to resend" + ); + + // The refused send consumed no sequence: the next frame is delivered in + // order instead of waiting behind a gap that never fills. + hub.send(&session, "schema", b"next".to_vec()).unwrap(); + let mut delivered = None; + for _ in 0..20 { + if let Some(frame) = peer.receive_once().unwrap() { + delivered = Some(frame); + break; + } + } + assert_eq!(delivered.expect("the next frame arrives").payload, b"next"); +} + +#[test] +fn a_send_that_fails_permanently_after_a_fragment_left_ends_the_session() { + let (mut hub, _peer, session) = one_peer(Some(1000)); + hub.unsendable_after.insert(session.remote_addr, 1); + + let error = hub + .send(&session, "schema", vec![7; 3000]) + .expect_err("the second fragment can never be sent"); + assert!(is_permanent_send_error( + error.downcast_ref::().unwrap() + )); + assert!(hub.sessions().is_empty()); + let CultNetRudpServerEvent::Disconnected { reason, .. } = next_event(&mut hub) else { + panic!("the caller sees the session end"); + }; + assert!(is_unsendable_reason(&reason)); +} + +#[test] +fn a_permanent_failure_in_resends_ends_only_that_peers_session() { + let (mut hub, (mut x, x_session), (mut y, y_session)) = two_peers(); + // Both peers' first datagram is lost, so both have a reliable resend due. + hub.failing_peers.insert(x_session.remote_addr); + hub.failing_peers.insert(y_session.remote_addr); + hub.send(&x_session, "schema", b"for-x".to_vec()).unwrap(); + hub.send(&y_session, "schema", b"for-y".to_vec()).unwrap(); + + // X's datagram can now never be sent; Y's path recovers. X is visited first. + hub.failing_peers.clear(); + hub.unsendable_after.insert(x_session.remote_addr, 0); + std::thread::sleep(Duration::from_millis(40)); + hub.poll_resends().unwrap(); + + assert_eq!(hub.sessions(), vec![y_session.clone()]); + let CultNetRudpServerEvent::Disconnected { session, reason } = next_event(&mut hub) else { + panic!("the caller sees the session end"); + }; + assert_eq!(session, x_session); + assert!(is_unsendable_reason(&reason)); + let mut delivered = None; + for _ in 0..20 { + if let Some(frame) = y.receive_once().unwrap() { + delivered = Some(frame); + break; + } + } + assert_eq!(delivered.expect("Y receives its resend").payload, b"for-y"); + // The goodbye reaches X and names the error. + for _ in 0..20 { + x.receive_once().unwrap(); + if x.disconnect_reason().is_some() { + break; + } + } + assert!(is_unsendable_reason( + x.disconnect_reason().expect("X is told its session ended") + )); +} + +#[test] +fn a_permanent_failure_replying_to_a_frame_ends_the_session_after_the_frame_is_delivered() { + let (mut hub, (mut x, x_session), (_y, _y_session)) = two_peers(); + hub.unsendable_after.insert(x_session.remote_addr, 0); + x.send("schema", b"from-x".to_vec()).unwrap(); + + let CultNetRudpServerEvent::Frame { frame, .. } = next_event(&mut hub) else { + panic!("the frame is delivered before the session ends"); + }; + assert_eq!(frame.payload, b"from-x"); + let CultNetRudpServerEvent::Disconnected { session, reason } = next_event(&mut hub) else { + panic!("the session ends"); + }; + assert_eq!(session, x_session); + assert!(is_unsendable_reason(&reason)); + assert_eq!(hub.sessions().len(), 1); +} + +#[test] +fn a_permanent_failure_answering_a_repeated_connect_ends_the_session() { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let raw = socket(); + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + initial_sequence: Some(5), + resend_delay_ms: 10, + max_pending_reliable_packets: None, + }); + let wire = encode_rudp_packet(&session.create_connect(0, Vec::new()).unwrap()).unwrap(); + + raw.send_to(&wire, server_addr).unwrap(); + let CultNetRudpServerEvent::Connected { session: admitted } = next_event(&mut hub) else { + panic!("the first Connect is admitted"); + }; + hub.unsendable_after.insert(raw.local_addr().unwrap(), 0); + raw.send_to(&wire, server_addr).unwrap(); + + let CultNetRudpServerEvent::Disconnected { + session: ended, + reason, + } = next_event(&mut hub) + else { + panic!("the reply to the repeat can never be sent, so the session ends"); + }; + assert_eq!(ended, admitted); + assert!(is_unsendable_reason(&reason)); + assert!(hub.sessions().is_empty()); +} + +#[test] +fn a_connect_whose_accept_can_never_be_sent_starts_no_session() { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let mut peer = client(server_addr); + hub.unsendable_after + .insert(peer.socket.local_addr().unwrap(), 0); + + peer.connect(Vec::new()).unwrap(); + for _ in 0..5 { + assert_eq!(hub.receive_event_once().unwrap(), None); + } + assert!(hub.sessions().is_empty()); + assert_eq!(hub.stats().packets_dropped, 1); +} + +#[test] +fn a_healthy_send_counts_its_bytes_and_no_failure() { + let (mut hub, mut peer, session) = one_peer(None); + let before = hub.stats().bytes_sent; + let received_before = peer.stats().bytes_received; + hub.send(&session, "schema", b"hello".to_vec()).unwrap(); + for _ in 0..20 { + if peer.receive_once().unwrap().is_some() { + break; + } + } + let sent = hub.stats().bytes_sent - before; + assert!(sent > b"hello".len() as u64); + assert_eq!(sent, peer.stats().bytes_received - received_before); + assert_eq!(hub.stats().send_failures, 0); +} + +#[test] +fn only_a_datagram_that_can_never_be_sent_is_a_permanent_failure() { + use std::io::{Error, ErrorKind}; + assert!(is_permanent_send_error(&Error::from(ErrorKind::InvalidInput))); + assert!(!is_permanent_send_error(&Error::other("no os code"))); + assert!(!is_permanent_send_error(&Error::from(ErrorKind::WouldBlock))); + assert!(!is_permanent_send_error(&Error::from(ErrorKind::ConnectionReset))); + #[cfg(target_os = "linux")] + let (permanent, transient) = ( + // EMSGSIZE, EAFNOSUPPORT, EINVAL + [90, 97, 22], + // EPERM, EINTR, EAGAIN, EACCES, ENETDOWN, ENETUNREACH, EADDRNOTAVAIL, ENOBUFS, ECONNREFUSED, EHOSTUNREACH + [1, 4, 11, 13, 100, 101, 99, 105, 111, 113], + ); + #[cfg(windows)] + let (permanent, transient) = ( + // WSAEMSGSIZE, WSAEAFNOSUPPORT, WSAEINVAL + [10040, 10047, 10022], + // WSAEACCES, WSAEWOULDBLOCK, WSAENETDOWN, WSAENETUNREACH, WSAEADDRNOTAVAIL, WSAENOBUFS, WSAECONNRESET, + // WSAECONNREFUSED, WSAEHOSTUNREACH, WSAEINTR + [10013, 10035, 10050, 10051, 10049, 10055, 10054, 10061, 10065, 10004], + ); + #[cfg(any(target_os = "linux", windows))] + { + for code in permanent { + assert!( + is_permanent_send_error(&Error::from_raw_os_error(code)), + "os error {code} is permanent" + ); + } + for code in transient { + assert!( + !is_permanent_send_error(&Error::from_raw_os_error(code)), + "os error {code} is transient" + ); + } + } +} + +#[test] +fn a_permanent_failure_sending_a_disconnect_is_returned_to_the_caller() { + let (mut hub, _peer, session) = one_peer(None); + hub.unsendable_after.insert(session.remote_addr, 0); + + let error = hub + .disconnect(&session, b"bye".to_vec()) + .expect_err("the goodbye can never be sent"); + assert!(is_permanent_send_error( + error.downcast_ref::().unwrap() + )); +} + +#[test] +fn a_peer_ended_in_the_resend_loop_is_sent_nothing_more_from_it() { + let (mut hub, (_x, x_session), (_y, _y_session)) = two_peers(); + hub.failing_peers.insert(x_session.remote_addr); + hub.send(&x_session, "schema", b"one".to_vec()).unwrap(); + hub.send(&x_session, "schema", b"two".to_vec()).unwrap(); + assert_eq!(hub.stats().send_failures, 2); + + hub.unsendable_after.insert(x_session.remote_addr, 0); + std::thread::sleep(Duration::from_millis(40)); + hub.poll_resends().unwrap(); + + // Both resends are due. The first can never be sent and ends the session; + // the goodbye is the only other datagram attempted at X. + assert_eq!(hub.stats().send_failures, 3); + assert_eq!(hub.sessions().len(), 1); +} + +#[test] +fn a_refused_send_withdraws_the_fragments_the_window_queued_as_well() { + let (mut hub, _peer, session) = one_peer(Some(1)); + let outstanding = hub.peers[&session.remote_addr] + .session + .outstanding_reliable_packet_count(); + hub.unsendable_after.insert(session.remote_addr, 0); + + // 1,100 one-byte fragments: the flow window admits the first ones and + // queues the rest, and the first datagram can never be sent. + hub.send(&session, "schema", vec![7; 1_100]) + .expect_err("the first fragment can never be sent"); + + assert_eq!( + hub.peers[&session.remote_addr] + .session + .outstanding_reliable_packet_count(), + outstanding, + "nothing of the refused send stays queued" + ); +} + +fn receive_frame( + peer: &mut CultNetRudpSocketTransportConnection, + hub: &mut CultNetRudpServerHub, +) -> Option> { + for _ in 0..600 { + if let Some(frame) = peer.receive_once().unwrap() { + return Some(frame.payload); + } + let _ = hub.receive_event_once().unwrap(); + hub.poll_resends().unwrap(); + } + None +} + +/// Wherever in a fragmented send the datagram that can never be sent falls, the +/// peer never delivers a frame built from it. At the first fragment nothing left, +/// the send is withdrawn, and the next send is delivered whole; later, the peer has +/// seen part of it, and the session ends with nothing delivered. +#[test] +fn a_send_refused_at_any_fragment_delivers_nothing_of_it() { + for fragments in [5_usize, 40] { + for failed in 0..fragments { + let (mut hub, mut peer, session) = one_peer(Some(1000)); + hub.send(&session, "schema", b"warm".to_vec()).unwrap(); + assert_eq!(receive_frame(&mut peer, &mut hub).as_deref(), Some(&b"warm"[..])); + let refused: Vec = (0..fragments * 1000).map(|i| (i % 251) as u8).collect(); + hub.unsendable_after.insert(session.remote_addr, failed); + let result = hub.send(&session, "schema", refused); + if hub.unsendable_after.remove(&session.remote_addr).is_some() { + // The flow window queued that fragment rather than sending it now. + assert!(result.is_ok(), "{fragments}/{failed}: not reached, yet refused"); + continue; + } + assert!(result.is_err(), "{fragments}/{failed}"); + if failed == 0 { + assert_eq!(hub.sessions().len(), 1, "{fragments}/{failed}"); + let next: Vec = (0..fragments * 1000).map(|i| ((i % 13) + 100) as u8).collect(); + hub.send(&session, "schema", next.clone()).unwrap(); + let delivered = receive_frame(&mut peer, &mut hub).expect("the next send arrives"); + assert!(delivered == next, "{fragments}/{failed}: delivered something else"); + for _ in 0..30 { + if let Some(frame) = peer.receive_once().unwrap() { + panic!("{fragments}/{failed}: a stray frame of {} bytes", frame.payload.len()); + } + let _ = hub.receive_event_once().unwrap(); + hub.poll_resends().unwrap(); + } + } else { + assert!(hub.sessions().is_empty(), "{fragments}/{failed}"); + for _ in 0..30 { + match peer.receive_once() { + Ok(Some(frame)) => panic!( + "{fragments}/{failed}: the peer delivered {} bytes", + frame.payload.len() + ), + Ok(None) => {} + Err(_) => break, + } + } + } + } + } +} diff --git a/packages/cultnet-rs/src/transport.rs b/packages/cultnet-rs/src/transport.rs index 71e714c4..f2c196d4 100644 --- a/packages/cultnet-rs/src/transport.rs +++ b/packages/cultnet-rs/src/transport.rs @@ -31,6 +31,11 @@ pub struct CultNetTransportStats { /// unadmitted sender, or a packet the session refused. A steady rate means /// a flow is being moved or scanned, not that this transport is failing. pub packets_dropped: u64, + /// Datagrams a multi-peer server could not send to one peer. Each is a lost + /// datagram attributed to that peer: reliable packets stay pending and are + /// resent, and no other peer's poll is affected. Zero for single-peer + /// transports, whose callers get the send error itself. + pub send_failures: u64, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index 9309b05b..1e856822 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -106,6 +106,8 @@ export { createRudpTransportProfile, decodeRudpPacket, encodeRudpPacket, + isPermanentSendError, + sendRudpDatagram, rudpClientBindHost, CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS, CultNetRudpReconnectLoop, diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index fa116c07..90760591 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -12,6 +12,8 @@ import { CultNetRudpSocketTransportConnection, decodeRudpPacket, encodeRudpPacket, + isPermanentSendError, + sendRudpDatagram, rudpClientBindHost, type CultNetRudpPacket, } from "./rudp"; @@ -39,6 +41,12 @@ export interface CultNetOperationServer { readonly endpoint: string; /** Datagrams read and discarded: malformed, unadmitted, refused by their session, or failed in handling. */ readonly packetsDropped: number; + /** + * Datagrams that could not be sent to a peer and may yet be (no route, full buffers): + * each is that peer's lost datagram, resent if it was reliable. One that can never be + * sent as built ends that peer's session instead and is not counted here. + */ + readonly sendFailures: number; close(): Promise; } @@ -56,8 +64,12 @@ interface RemoteSession { handling: number; /** When the last handler finished: the response still needs its acknowledgement. */ lastHandledAtMs: number; + /** A packet it owed could never be sent, so the session ended. */ + unsendable: boolean; } +type SendPacket = (key: string, peer: RemoteSession, packet: CultNetRudpPacket) => void; + export async function startCultNetOperationServer( options: CultNetOperationServerOptions, ): Promise { @@ -66,16 +78,32 @@ export async function startCultNetOperationServer( const socket = createSocket("udp4"); const connectionId = options.connectionId ?? DEFAULT_CONNECTION_ID; const sessions = new Map(); - const sendPacket = (remote: RemoteInfo, packet: CultNetRudpPacket): void => { - const wire = encodeRudpPacket(packet); - socket.send(wire, remote.port, remote.address); - }; let packetsDropped = 0; + let sendFailures = 0; + const sendPacket: SendPacket = (key, peer, packet) => { + const generation = peer.session.generation; + sendRudpDatagram(socket, encodeRudpPacket(packet), peer.remote.port, peer.remote.address, (error) => { + if (!isPermanentSendError(error)) { + sendFailures += 1; + return; + } + // A failure reported after the generation that sent the datagram ended is owed by + // nothing: that generation's end already told the peer, and a client that connected + // again from the endpoint owns it now. A live generation is the one in `sessions`. + if (peer.session.generation !== generation) return; + // The datagram can never be sent as built: the session that owes it ends, never + // the server, and the peer is told. + peer.unsendable = true; + sessions.delete(key); + const goodbye = encodeRudpPacket(peer.session.endUnsendableSession(error)); + sendRudpDatagram(socket, goodbye, peer.remote.port, peer.remote.address, () => {}); + }); + }; socket.on("message", (wire, remote) => { // What a datagram carries is the sender's business: a rejection here must // drop and count the packet, never become an unhandled rejection that ends // the process. - handleServerDatagram(sessions, connectionId, options, wire, remote, sendPacket).then( + handleServerDatagram(sessions, connectionId, options, wire, remote, socket, sendPacket).then( admitted => { if (!admitted) packetsDropped += 1; }, () => { packetsDropped += 1; }, ); @@ -90,8 +118,8 @@ export async function startCultNetOperationServer( if (peer.handling > 0 || now - peer.lastHandledAtMs < idleTimeoutMs) continue; if (peer.session.checkTimeout(now, idleTimeoutMs)) sessions.delete(key); } - for (const peer of sessions.values()) { - for (const packet of peer.session.dueResends(Date.now())) sendPacket(peer.remote, packet); + for (const [key, peer] of sessions) { + for (const packet of peer.session.dueResends(Date.now())) sendPacket(key, peer, packet); } }, 25); resendTimer.unref?.(); @@ -100,6 +128,7 @@ export async function startCultNetOperationServer( return { endpoint, get packetsDropped() { return packetsDropped; }, + get sendFailures() { return sendFailures; }, close: async () => { clearInterval(resendTimer); await closeSocket(socket); @@ -155,7 +184,8 @@ async function handleServerDatagram( options: CultNetOperationServerOptions, wire: Buffer, remote: RemoteInfo, - sendPacket: (remote: RemoteInfo, packet: CultNetRudpPacket) => void, + socket: Socket, + sendPacket: SendPacket, ): Promise { let packet: CultNetRudpPacket; try { @@ -167,14 +197,24 @@ async function handleServerDatagram( const key = `${remote.address}:${remote.port}`; let peer = sessions.get(key); if (packet.packetType === "connect") { - if (!peer) { - if (sessions.size >= MAX_OPERATION_SESSIONS) return false; - peer = { session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), remote, handling: 0, lastHandledAtMs: 0 }; - sessions.set(key, peer); + if (peer) { + // A Connect from an admitted peer repeats: the session answers with the + // Accept already owed and queues nothing. + sendPacket(key, peer, peer.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); + return true; } - // A Connect from an admitted peer repeats: the session answers with the - // Accept already owed and queues nothing. - sendPacket(remote, peer.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); + if (sessions.size >= MAX_OPERATION_SESSIONS) return false; + const candidate: RemoteSession = { + session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), + remote, + handling: 0, + lastHandledAtMs: 0, + unsendable: false, + }; + sendPacket(key, candidate, candidate.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); + // A peer whose Accept can never be sent cannot be answered, so no session starts. + if (candidate.unsendable) return false; + sessions.set(key, candidate); return true; } if (!peer) return false; @@ -182,15 +222,20 @@ async function handleServerDatagram( try { result = peer.session.receive(packet, Date.now()); } catch { - endSession(sessions, key, peer, sendPacket); + endSession(sessions, key, peer, socket); return false; } - if (result.reply) sendPacket(remote, result.reply); - for (const ready of result.readyToSend ?? []) sendPacket(remote, ready); + if (result.reply) sendPacket(key, peer, result.reply); + for (const ready of result.readyToSend ?? []) sendPacket(key, peer, ready); if (result.disconnected) { sessions.delete(key); return true; } + // A request belongs to the session generation it arrived on. If that generation ends while a + // handler runs (a goodbye, a refusal, a timeout, a new Connect from the same endpoint), what + // the handler returns or throws belongs to no live session, and nothing of it may reach the + // endpoint: a client that connected again there owns it now. + const generation = peer.session.generation; for (const frame of result.delivered) { if (frame.channelId !== "schema") continue; try { @@ -204,22 +249,24 @@ async function handleServerDatagram( peer.handling -= 1; peer.lastHandledAtMs = Date.now(); } + if (peer.session.generation !== generation) return true; const payload = encode(encodeCultNetMessageForWire(response, "cultnet.schema.v0")); for (const responsePacket of peer.session.sendMany("schema", payload, { reliable: true, ordered: true, nowMs: Date.now(), maxFragmentBytes: options.maxFragmentBytes ?? 2048, - })) sendPacket(remote, responsePacket); + })) sendPacket(key, peer, responsePacket); } catch { + if (peer.session.generation !== generation) return true; // The session recorded this request's sequence, so keeping it would // acknowledge the retransmit of a request that was never handled. - endSession(sessions, key, peer, sendPacket); + endSession(sessions, key, peer, socket); return false; } } if (packet.packetType === "data" || result.delivered.length > 0) { - sendPacket(remote, peer.session.createAckForReceived(packet.sequence)); + sendPacket(key, peer, peer.session.createAckForReceived(packet.sequence)); } return true; } @@ -227,17 +274,19 @@ async function handleServerDatagram( /** * Ends a session that took a packet it could not serve, and tells the peer. The * reset comes first, or the goodbye's ack field would acknowledge the very - * packet that was refused. A session already replaced under the same key is left. + * packet that was refused. The goodbye is best-effort: the session ends whether + * or not the peer hears it. */ function endSession( sessions: Map, key: string, peer: RemoteSession, - sendPacket: (remote: RemoteInfo, packet: CultNetRudpPacket) => void, + socket: Socket, ): void { - if (sessions.get(key) === peer) sessions.delete(key); + sessions.delete(key); peer.session.resetPeerState(); - sendPacket(peer.remote, peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + const goodbye = encodeRudpPacket(peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + sendRudpDatagram(socket, goodbye, peer.remote.port, peer.remote.address, () => {}); } function parseRudpEndpoint(endpoint: string): { host: string; port: number } { diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index 5c1c9c5e..e68293d3 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -30,6 +30,10 @@ import { const RUDP_MAGIC = [0x43, 0x4e, 0x52, 0x30] as const; // CNR0 const RUDP_VERSION = 0; const RUDP_FIXED_HEADER_BYTES = 36; +/// The largest UDP payload a datagram can carry: 65,535 less the UDP header and, +/// over IPv4, the minimal IP header. IPv6's header is outside its payload length. +const MAX_UDP4_DATAGRAM_BYTES = 65_507; +const MAX_UDP6_DATAGRAM_BYTES = 65_527; const MAX_CHANNEL_ID_BYTES = 255; export const CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS = 32; const RUDP_RECEIVED_SEQUENCE_WINDOW = 4_096; @@ -668,6 +672,18 @@ export class CultNetRudpSession { }); } + /// Ends a session that owes its peer a packet that can never be sent as + /// built (see `isPermanentSendError`). Resending it would fail forever, and + /// dropping it would leave a reliable sequence the peer waits on for good, so + /// the session cannot be kept. Returns the goodbye for the peer; its reason + /// names the error's code, never its message, which can quote the address. + endUnsendableSession(error: Error): CultNetRudpPacket { + this.resetPeerState(); + return this.createDisconnect( + Buffer.from(`${RUDP_UNSENDABLE_PACKET_REASON}: ${sendErrorCode(error)}`, "utf8"), + ); + } + checkTimeout(nowMs: number, timeoutMs: number): boolean { if (!this.#connected || this.#lastReceivedAtMs === undefined) { return false; @@ -1025,6 +1041,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement readonly #mode: "client" | "server"; readonly #resendTimer: NodeJS.Timeout; readonly #maxFragmentBytes: number | undefined; + readonly #maxDatagramBytes: number; #remoteHost: string | undefined; #remotePort: number | undefined; #closed = false; @@ -1037,6 +1054,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement framesReceived: 0, framesSent: 0, packetsDropped: 0, + sendFailures: 0, }; constructor(options: CultNetRudpSocketTransportOptions) { @@ -1055,6 +1073,8 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement const address = this.#socket.address(); const localPort = typeof address === "string" ? undefined : address.port; const localHost = typeof address === "string" ? undefined : address.address; + this.#maxDatagramBytes = + typeof address !== "string" && String(address.family).endsWith("6") ? MAX_UDP6_DATAGRAM_BYTES : MAX_UDP4_DATAGRAM_BYTES; this.profile = createRudpTransportProfile(options.runtimeId, { transportId: options.transportId, host: localHost, @@ -1096,6 +1116,15 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } send(channelId: string, payload: Uint8Array): void { + // A datagram larger than UDP can carry can never be sent, and Node reports that + // only later, to the send's callback. Checked here it is the caller's error, as + // in the other runtimes: the send throws, and nothing of it is queued or takes a + // sequence. Every datagram of a send is at most one fragment of its payload. + const largestPayload = + this.#maxFragmentBytes === undefined ? payload.byteLength : Math.min(payload.byteLength, this.#maxFragmentBytes); + if (RUDP_FIXED_HEADER_BYTES + Buffer.byteLength(channelId, "utf8") + largestPayload > this.#maxDatagramBytes) { + throw Object.assign(new Error("RUDP datagram is larger than UDP can carry."), { code: "EMSGSIZE" }); + } const packets = this.#session.sendMany(channelId, payload, { ...channelOptions(channelId), nowMs: Date.now(), @@ -1170,11 +1199,22 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement // The goodbye is built after the reset, or its ack field would acknowledge // the very frame the session refused. this.#session.resetPeerState(); + this.#endWithGoodbye(this.#session.createDisconnect(reason)); + } + + /** The session owes its peer a packet that can never be sent as built. */ + #endUnsendableSession(error: Error): void { + this.#endWithGoodbye(this.#session.endUnsendableSession(error)); + } + + #endWithGoodbye(goodbye: CultNetRudpPacket): void { + const reason = goodbye.payload; this.#endedReason = reason; - try { - this.#sendPacket(this.#session.createDisconnect(reason)); - } catch { + if (this.#remoteHost && this.#remotePort !== undefined) { // Best-effort: the session ends whether or not the peer hears it. + const wire = encodeRudpPacket(goodbye); + this.#stats.bytesSent += wire.length; + sendRudpDatagram(this.#socket, wire, this.#remotePort, this.#remoteHost, () => {}); } if (this.#mode === "server") { // Only a Connect can claim the endpoint again. @@ -1245,7 +1285,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement this.#stats.packetsDropped += 1; return; } - this.#sendPacket(accept); + this.#sendInPoll(accept); return; } @@ -1262,10 +1302,10 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } try { if (result.reply) { - this.#sendPacket(result.reply); + this.#sendInPoll(result.reply); } for (const ready of result.readyToSend ?? []) { - this.#sendPacket(ready); + this.#sendInPoll(ready); } if (result.pong) { this.emit("pong", { payload: result.pongPayload ?? new Uint8Array() }); @@ -1284,7 +1324,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement return; } if (packet.packetType === "accept" || packet.packetType === "data" || result.delivered.length > 0) { - this.#sendPacket(this.#session.createAckForReceived(packet.sequence)); + this.#sendInPoll(this.#session.createAckForReceived(packet.sequence)); } } catch (error) { this.emit("error", error instanceof Error ? error : new Error(String(error))); @@ -1292,18 +1332,63 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } #sendDueResends(): void { + const generation = this.#session.generation; for (const packet of this.#session.dueResends(Date.now())) { - this.#sendPacket(packet); + this.#sendInPoll(packet); + // A resend that can never be sent ended the session and what it owed. + if (this.#session.generation !== generation) return; } } + /** A caller-directed send: a failure the send throws is the caller's error. */ #sendPacket(packet: CultNetRudpPacket): void { + const thrown = this.#transmit(packet); + if (thrown) throw thrown; + } + + /** + * A send with no caller to hand a failure to (a receive or a resend): the failure + * is handled as every later one is, and never escapes the timer or the socket's + * "message" handler. + */ + #sendInPoll(packet: CultNetRudpPacket): void { + const thrown = this.#transmit(packet); + if (thrown) this.#sendFailed(thrown); + } + + /** Sends one packet and returns the failure the send threw, if any. */ + #transmit(packet: CultNetRudpPacket): Error | undefined { if (!this.#remoteHost || this.#remotePort === undefined) { throw new Error("RUDP socket transport does not have a remote endpoint."); } const wire = encodeRudpPacket(packet); this.#stats.bytesSent += wire.length; - this.#socket.send(wire, this.#remotePort, this.#remoteHost); + let thrown: Error | undefined; + let sending = true; + const generation = this.#session.generation; + sendRudpDatagram(this.#socket, wire, this.#remotePort, this.#remoteHost, (error) => { + if (sending) { + thrown = error; + } else if (this.#session.generation === generation) { + // A later failure belongs to the session generation that sent it. + this.#sendFailed(error); + } + }); + sending = false; + return thrown; + } + + /** + * A datagram that can never be sent ends the session; any other failure is a lost + * datagram, counted and never emitted. Node reports a failed send only to its callback, + * not on the socket's "error" event, and an "error" with no listener ends the process. + */ + #sendFailed(error: Error): void { + if (isPermanentSendError(error)) { + this.#endUnsendableSession(error); + return; + } + this.#stats.sendFailures += 1; } } @@ -1397,6 +1482,61 @@ export class CultNetRudpReconnectLoop extends EventEmitter { } } +/** Prefix of the goodbye reason for a session that owed a packet which can never be sent. */ +const RUDP_UNSENDABLE_PACKET_REASON = "packet could not be sent"; + +/** + * Failures that can never pass for the datagram as built: too large for a datagram + * (`EMSGSIZE`), an address malformed for the socket (`EINVAL`, and Node's own + * `ERR_SOCKET_BAD_PORT`, which it throws for port 0 before the system would say + * `EINVAL`), or an address of another family (`EAFNOSUPPORT`). libuv names these the + * same on every platform. + */ +const PERMANENT_SEND_ERROR_CODES: ReadonlySet = new Set([ + "EMSGSIZE", + "EINVAL", + "EAFNOSUPPORT", + "ERR_SOCKET_BAD_PORT", +]); + +function sendErrorCode(error: Error): string { + const code = (error as NodeJS.ErrnoException).code; + return typeof code === "string" && code.length > 0 ? code : "UNKNOWN"; +} + +/** + * Whether a failed UDP send can never succeed for the datagram as built, so resending it is + * pointless and the session that owes it cannot be kept. Everything else (no route, full + * buffers, a closed socket, an interrupted call) is a property of the path or the moment: a + * lost datagram, not a reason to end a session. + */ +export function isPermanentSendError(error: Error): boolean { + return PERMANENT_SEND_ERROR_CODES.has(sendErrorCode(error)); +} + +/** + * Sends one datagram. Node reports a failed send two ways: it throws for an argument it + * refuses (port 0 among them) and passes a system failure to the callback later. Both + * reach `onFailure` and nothing else, so a datagram that cannot be sent to one peer never + * escapes a timer or a receive handler, and never reaches the socket's "error" event. A + * failure the send throws reaches `onFailure` before this returns. + */ +export function sendRudpDatagram( + socket: Socket, + wire: Uint8Array, + port: number, + address: string, + onFailure: (error: Error) => void, +): void { + try { + socket.send(wire, port, address, (error) => { + if (error) onFailure(error); + }); + } catch (error) { + onFailure(error instanceof Error ? error : new Error(String(error))); + } +} + export function createRudpTransportProfile( runtimeId: string, options: RudpTransportProfileOptions = {}, diff --git a/packages/cultnet-ts/src/transport.ts b/packages/cultnet-ts/src/transport.ts index f900aa23..02f5e8c1 100644 --- a/packages/cultnet-ts/src/transport.ts +++ b/packages/cultnet-ts/src/transport.ts @@ -16,6 +16,13 @@ export interface CultNetTransportStats { framesSent: number; /** Datagrams read and discarded because they belong to no session on this transport. */ packetsDropped: number; + /** + * Datagrams this transport could not send and treats as lost (no route, full buffers): + * a reliable packet stays pending and is resent. A send a caller made throws its failure + * to the caller instead, and a datagram that can never be sent as built ends the session. + * Zero for stream transports. + */ + sendFailures: number; } export interface CultNetReconnectPolicy { @@ -195,6 +202,7 @@ export class TcpFramedTransportConnection extends EventEmitter implements CultNe framesReceived: 0, framesSent: 0, packetsDropped: 0, + sendFailures: 0, }; constructor(stream: Duplex, profile: CultNetTransportProfile) { diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index e6b62bdb..a6f19800 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -2825,6 +2825,89 @@ test("server-mode transport admits a restarted client and keeps its session when } }); +// A request belongs to the session generation it arrived on. A client that connects again +// from the same endpoint while a handler runs owns that endpoint now: nothing the handler +// returns or throws for the old generation may reach it. +async function restartDuringHandler(settle: (release: { resolve: () => void; reject: (error: Error) => void }) => void) { + let release: { resolve: () => void; reject: (error: Error) => void } | undefined; + const server = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + handler: async (request) => { + if (request.messageId === "held") { + await new Promise((resolve, reject) => { release = { resolve, reject: (error) => reject(error) }; }); + } + return { + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: "gamecult.eve.plugin_abi.response.v1", + payloadEncoding: "messagepack-base64", + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "sai-sidecar", + }; + }, + }); + const connectionId = 0x43554c54; + const port = Number(new URL(server.endpoint).port); + const socket = await bindUdpSocket(); + let session = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const answered: string[] = []; + const goodbyes: string[] = []; + socket.on("message", (wire) => { + const packet = decodeRudpPacket(wire); + if (packet.packetType === "disconnect") goodbyes.push(Buffer.from(packet.payload ?? []).toString("utf8")); + const result = session.receive(packet, Date.now()); + if (packet.packetType === "accept" || packet.reliable) { + socket.send(encodeRudpPacket(session.createAckForReceived(packet.sequence)), port, "127.0.0.1"); + } + for (const frame of result.delivered) { + answered.push((decodeMsgpack(frame.payload) as { messageId: string }).messageId); + } + }); + const toServer = (packet: CultNetRudpPacket) => socket.send(encodeRudpPacket(packet), port, "127.0.0.1"); + const request = (messageId: string) => Buffer.from(encode(encodeCultNetMessageForWire({ + schemaVersion: "cultnet.operation_request.v0", + messageId, + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + } satisfies CultNetOperationRequestMessage))); + try { + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the first Accept"); + toServer(session.send("schema", request("held"), { reliable: true, ordered: true })); + await waitFor(() => release !== undefined, "the held handler"); + + session = new CultNetRudpSession({ connectionId, initialSequence: 7 }); + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the restarted client's Accept"); + settle(release!); + await new Promise((resolve) => setTimeout(resolve, 100)); + toServer(session.send("schema", request("after-restart"), { reliable: true, ordered: true })); + await waitFor(() => answered.includes("after-restart"), "the restarted client's answer"); + + assert.deepEqual(answered, ["after-restart"], "the old generation's response reached the new session"); + assert.deepEqual(goodbyes, [], "a goodbye for the old generation reached the new session"); + assert.equal(session.connected, true); + } finally { + socket.close(); + await server.close(); + } +} + +test("an operation response for a generation a new Connect replaced is not sent into the new session", async () => { + await restartDuringHandler((release) => release.resolve()); +}); + +test("an operation handler that fails after a new Connect replaced its generation says no goodbye to the new session", async () => { + await restartDuringHandler((release) => release.reject(new Error("handler failed"))); +}); + test("RUDP operation service admits a restarted client on the same address", async () => { const server = await startCultNetOperationServer({ runtimeId: "sai-sidecar", diff --git a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts index 756ef97d..598d6fe6 100644 --- a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts +++ b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts @@ -61,13 +61,6 @@ const cultMeshTsRoot = resolve(cultLibRoot, "packages", "cultmesh-ts"); const cultMeshQuicPeerScript = resolve(cultMeshTsRoot, "dist-test", "test", "interop", "cultmesh-quic-peer.js"); const quicNativeTestsProject = resolve(cultLibRoot, "tests", "GameCult.Mesh.Quic.Native.Tests", "GameCult.Mesh.Quic.Native.Tests.csproj"); const quicManagedTestsProject = resolve(cultLibRoot, "tests", "GameCult.Mesh.Quic.Tests", "GameCult.Mesh.Quic.Tests.csproj"); -const rustBinaryPath = resolve( - cultnetRsRoot, - "target", - "debug", - "examples", - process.platform === "win32" ? "cultnet_interop_peer.exe" : "cultnet_interop_peer", -); const discoveryGroup = "239.77.44.11"; const serveReadyTimeoutMs = 90_000; @@ -78,6 +71,67 @@ let csharpInteropPeerBuild: Promise | undefined; let kotlinInteropPeerBuild: Promise | undefined; let cultMeshTsInteropPeerBuild: Promise | undefined; +let rustInteropPeerPathCache: string | undefined; +let tsInteropPeerChecked = false; +let pythonInteropChecked = false; + +// Cargo decides where it builds: CARGO_TARGET_DIR, a config's build.target-dir, or the crate's own +// target directory. The peer's path is asked of cargo, not assumed. +function rustInteropPeerPath(): string { + if (rustInteropPeerPathCache !== undefined) { + return rustInteropPeerPathCache; + } + const metadata = spawnSync(cargoCommand, ["metadata", "--format-version", "1", "--no-deps"], { + cwd: cultnetRsRoot, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + if (metadata.status !== 0) { + throw new Error( + `The Rust interop peer's target directory is unknown: \`cargo metadata\` failed in ${cultnetRsRoot}.\n${metadata.stderr ?? metadata.error?.message ?? ""}`, + ); + } + const targetDirectory = (JSON.parse(metadata.stdout) as { target_directory: string }).target_directory; + rustInteropPeerPathCache = resolve( + targetDirectory, + "debug", + "examples", + process.platform === "win32" ? "cultnet_interop_peer.exe" : "cultnet_interop_peer", + ); + return rustInteropPeerPathCache; +} + +// The TypeScript peer imports @gamecult/cultcache-ts from its built output. Unbuilt, the peer dies +// on import and a lane fails with a spawn or timeout error that names neither. +function tsInteropPeerScript(): string { + if (!tsInteropPeerChecked) { + try { + require.resolve("@gamecult/cultcache-ts"); + } catch { + throw new Error( + "@gamecult/cultcache-ts is not built, and the TypeScript interop peer imports it. Run `npm run build --workspace packages/cultcache-ts` first.", + ); + } + tsInteropPeerChecked = true; + } + return tsPeerScript; +} + +// The Python peers import msgpack. Missing, each Python lane fails with a traceback in a child's +// stderr instead of a reason. +function pythonInteropCommand(): string { + if (!pythonInteropChecked) { + const probe = spawnSync(pythonCommand, ["-c", "import msgpack"], { encoding: "utf8" }); + if (probe.status !== 0) { + throw new Error( + `${pythonCommand} cannot import msgpack, which the Python interop peers need. Install the Python packages first: \`python -m pip install -e packages/cultcache-py -e packages/cultnet-py -e packages/cultmesh-py\`.\n${probe.stderr ?? probe.error?.message ?? ""}`, + ); + } + pythonInteropChecked = true; + } + return pythonCommand; +} + function resolvePythonCommand(): string { const codexPythonCommand = join(homedir(), ".cache", "codex-runtimes", "codex-primary-runtime", "dependencies", "python", "python.exe"); const candidates = process.platform === "win32" @@ -743,7 +797,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "start rust peer"); servers.push(await spawnServeProcess("rust", { - command: rustBinaryPath, + command: rustInteropPeerPath(), args: [ "serve", "--runtime-id", "rust-peer", @@ -765,7 +819,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state servers.push(await spawnServeProcess("ts", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-peer", "--runtime-kind", "node", @@ -805,7 +859,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "start python peer"); servers.push(await spawnServeProcess("python", { - command: pythonCommand, + command: pythonInteropCommand(), args: [ "-m", "cultnet_py.interop_peer", "serve", @@ -827,7 +881,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript dials rust"); const tsDial = await runJsonCommand("ts-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-client", "--runtime-kind", "node", @@ -846,7 +900,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript dials typescript over rudp"); const tsRudpDial = await runJsonCommand("ts-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-rudp-client", "--runtime-kind", "node", @@ -864,7 +918,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(tsRudpDial.fireReceipt.accepted, true); logInteropPhase("schema-v0", "rust dials csharp"); - const rustDial = await runJsonCommand("rust-dial", rustBinaryPath, [ + const rustDial = await runJsonCommand("rust-dial", rustInteropPeerPath(), [ "dial", "--runtime-id", "rust-client", "--runtime-kind", "rust", @@ -899,7 +953,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(csharpDial.fireReceipt.accepted, true); logInteropPhase("schema-v0", "python dials rust"); - const pythonDial = await runJsonCommand("python-dial", pythonCommand, [ + const pythonDial = await runJsonCommand("python-dial", pythonInteropCommand(), [ "-m", "cultnet_py.interop_peer", "dial", "--runtime-id", "python-client", @@ -917,7 +971,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(pythonDial.fireReceipt.accepted, true); logInteropPhase("schema-v0", "python dials python over rudp"); - const pythonRudpDial = await runJsonCommand("python-rudp-dial", pythonCommand, [ + const pythonRudpDial = await runJsonCommand("python-rudp-dial", pythonInteropCommand(), [ "-m", "cultnet_py.interop_peer", "dial", "--runtime-id", "python-rudp-client", @@ -937,7 +991,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript dials python"); const tsDialPython = await runJsonCommand("ts-dial-python", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-python-client", "--runtime-kind", "node", @@ -1257,7 +1311,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript discovery probe"); await expectProbePeers("ts-probe", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "probe", "--runtime-id", "ts-prober", "--discovery-port", String(discoveryPort), @@ -1266,7 +1320,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state ], cultNetTsRoot, {}, expectedPeers); logInteropPhase("schema-v0", "rust discovery probe"); - await expectProbePeers("rust-probe", rustBinaryPath, [ + await expectProbePeers("rust-probe", rustInteropPeerPath(), [ "probe", "--runtime-id", "rust-prober", "--discovery-port", String(discoveryPort), @@ -1285,7 +1339,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state ], cultLibRoot, {}, expectedPeers); logInteropPhase("schema-v0", "python discovery probe"); - await expectProbePeers("python-probe", pythonCommand, [ + await expectProbePeers("python-probe", pythonInteropCommand(), [ "-m", "cultnet_py.interop_peer", "probe", "--runtime-id", "python-prober", @@ -1319,7 +1373,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD }); servers.push(await spawnServeProcess("rust-rudp", { - command: rustBinaryPath, + command: rustInteropPeerPath(), args: [ "serve", "--runtime-id", "rust-rudp-peer", @@ -1341,7 +1395,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD servers.push(await spawnServeProcess("ts-rust-rudp", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-rust-rudp-peer", "--runtime-kind", "node", @@ -1361,7 +1415,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD logInteropPhase("ts-rust-full-rudp", "typescript dials rust over rudp"); const tsRudpDial = await runJsonCommand("ts-rust-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-rust-rudp-client", "--runtime-kind", "node", @@ -1379,7 +1433,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD assert.equal(tsRudpDial.fireReceipt.accepted, true); logInteropPhase("ts-rust-full-rudp", "rust dials typescript over rudp"); - const rustRudpDial = await runJsonCommand("rust-rudp-dial", rustBinaryPath, [ + const rustRudpDial = await runJsonCommand("rust-rudp-dial", rustInteropPeerPath(), [ "dial", "--runtime-id", "rust-rudp-client", "--runtime-kind", "rust", @@ -2186,7 +2240,7 @@ test("CultNet TypeScript and C# full interop peers exchange schema-v0 over RUDP" servers.push(await spawnServeProcess("ts-csharp-rudp", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-csharp-rudp-peer", "--runtime-kind", "node", @@ -2206,7 +2260,7 @@ test("CultNet TypeScript and C# full interop peers exchange schema-v0 over RUDP" logInteropPhase("ts-csharp-full-rudp", "typescript dials csharp over rudp"); const tsRudpDial = await runJsonCommand("ts-csharp-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-csharp-rudp-client", "--runtime-kind", "node", @@ -2289,7 +2343,7 @@ test("CultNet TypeScript and Kotlin full interop peers exchange schema-v0 over R servers.push(await spawnServeProcess("ts-kotlin-rudp", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-kotlin-rudp-peer", "--runtime-kind", "node", @@ -2309,7 +2363,7 @@ test("CultNet TypeScript and Kotlin full interop peers exchange schema-v0 over R logInteropPhase("ts-kotlin-full-rudp", "typescript dials kotlin over rudp"); const tsRudpDial = await runJsonCommand("ts-kotlin-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-kotlin-rudp-client", "--runtime-kind", "node", @@ -2962,7 +3016,7 @@ interface RunningPythonRudpPeer { } function spawnPythonRudpPeer(options: RudpServerSpawnOptions = {}): RunningPythonRudpPeer { - const child = spawn(pythonCommand, ["-c", pythonRudpPeerScript], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpPeerScript], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3018,7 +3072,7 @@ function spawnPythonRudpPeer(options: RudpServerSpawnOptions = {}): RunningPytho } function spawnPythonRudpMessagePeer(): RunningPythonRudpPeer { - const child = spawn(pythonCommand, ["-c", pythonRudpMessagePeerScript], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpMessagePeerScript], { cwd: cultcachePyRoot, env: { ...process.env, PYTHONPATH: cultcachePySrc }, stdio: ["ignore", "pipe", "pipe"], @@ -3066,7 +3120,7 @@ function spawnPythonRudpMessagePeer(): RunningPythonRudpPeer { } function spawnRustRudpServer(options: RudpServerSpawnOptions = {}): RunningPythonRudpPeer { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-serve-once", "--bind-host", "127.0.0.1", ...(options.clientPayload ? ["--client-payload", options.clientPayload] : []), @@ -3124,7 +3178,7 @@ function spawnRustRudpServer(options: RudpServerSpawnOptions = {}): RunningPytho } function spawnRustRudpMessageServer(): RunningPythonRudpPeer { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-serve-message-once", "--bind-host", "127.0.0.1", ], { @@ -3405,7 +3459,7 @@ function spawnKotlinRudpMessageServer(): RunningPythonRudpPeer { } function spawnPythonRudpClient(remotePort: number): Omit { - const child = spawn(pythonCommand, ["-c", pythonRudpClientScript, String(remotePort)], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpClientScript, String(remotePort)], { cwd: cultcachePyRoot, env: { ...process.env, PYTHONPATH: cultcachePySrc }, stdio: ["ignore", "pipe", "pipe"], @@ -3420,7 +3474,7 @@ function spawnPythonRudpClient(remotePort: number): Omit { - const child = spawn(pythonCommand, ["-c", pythonRudpMessageClientScript, String(remotePort)], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpMessageClientScript, String(remotePort)], { cwd: cultcachePyRoot, env: { ...process.env, PYTHONPATH: cultcachePySrc }, stdio: ["ignore", "pipe", "pipe"], @@ -3517,7 +3571,7 @@ function spawnCSharpRudpMessageClient(remotePort: number): Omit { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-dial-once", "--target-host", "127.0.0.1", "--target-port", String(remotePort), @@ -3536,7 +3590,7 @@ function spawnRustRudpClient(remotePort: number): Omit { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-dial-message-once", "--target-host", "127.0.0.1", "--target-port", String(remotePort), @@ -3832,7 +3886,7 @@ async function runPythonCultMeshClient(port: number): Promise { " 'peers': [{'peerId': peer.peer_id, 'roles': list(peer.roles)} for peer in peers],", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3857,7 +3911,7 @@ async function runPythonCultNetRawClient(port: number): Promise { " 'shards': [{'shardId': shard.get('shardId'), 'epoch': shard.get('epoch')} for shard in shards.get('shards', [])],", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3894,7 +3948,7 @@ async function runPythonCultMeshNodeSync(port: number): Promise { " 'note': note,", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3959,7 +4013,7 @@ async function runPythonCultMeshNodeEmit(port: number): Promise { " 'note': note,", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -4013,7 +4067,7 @@ async function runPythonCultNetSubscription(port: number): Promise { " 'recordKey': change.get('document', {}).get('recordKey'),", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -4067,7 +4121,7 @@ async function runPythonCultMeshPredictionReconcile(port: number): Promise " 'note': note,", " }))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts new file mode 100644 index 00000000..4370d406 --- /dev/null +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -0,0 +1,520 @@ +// A datagram that cannot be sent to one peer is that peer's problem alone. Node throws +// for a send it refuses outright (a peer at UDP port 0, which any spoofer can claim) +// and reports a system failure later through the callback; neither may escape a +// timer or a receive handler, and only a datagram that can never be sent as built +// ends the session that owes it. +import assert from "node:assert/strict"; +import dgram, { type RemoteInfo, type Socket } from "node:dgram"; +import test from "node:test"; +import { encode } from "@msgpack/msgpack"; +import { + CultNetRudpSession, + CultNetRudpSocketTransportConnection, + decodeRudpPacket, + encodeCultNetMessageForWire, + encodeRudpPacket, + invokeCultNetOperation, + isPermanentSendError, + startCultNetOperationServer, + type CultNetOperationRequestMessage, + type CultNetOperationResponseMessage, +} from "../src"; + +async function bindUdpSocket(): Promise { + const socket = dgram.createSocket("udp4"); + await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); + return socket; +} + +function udpPort(socket: Socket): number { + const address = socket.address(); + assert.notEqual(typeof address, "string"); + return address.port; +} + +async function waitFor(predicate: () => boolean, description: string): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > 2_000) { + throw new Error(`Timed out waiting for ${description}.`); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** A Connect as it arrives from UDP source port 0: Node refuses to send anything back. */ +function connectFromPortZero(socket: Socket, connectionId: number): void { + const wire = Buffer.from(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(Date.now()))); + const remote: RemoteInfo = { address: "127.0.0.1", family: "IPv4", port: 0, size: wire.length }; + socket.emit("message", wire, remote); +} + +/** Records every exception that would otherwise end the process. */ +function watchUncaught(): { errors: unknown[]; stop: () => void } { + const errors: unknown[] = []; + const listener = (error: unknown) => errors.push(error); + process.on("uncaughtException", listener); + return { errors, stop: () => process.off("uncaughtException", listener) }; +} + +test("only a datagram that can never be sent as built is a permanent send failure", () => { + const failure = (code: string) => Object.assign(new Error("send failed"), { code }); + for (const code of ["EMSGSIZE", "EINVAL", "EAFNOSUPPORT", "ERR_SOCKET_BAD_PORT"]) { + assert.equal(isPermanentSendError(failure(code)), true, code); + } + for (const code of ["EHOSTUNREACH", "ENETUNREACH", "ECONNREFUSED", "ENOBUFS", "EAGAIN", "ERR_SOCKET_DGRAM_NOT_RUNNING"]) { + assert.equal(isPermanentSendError(failure(code)), false, code); + } + assert.equal(isPermanentSendError(new Error("no code")), false); +}); + +test("a server transport answering a Connect from port 0 ends that session and keeps serving", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203090; + const uncaught = watchUncaught(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + const errors: Error[] = []; + const reasons: string[] = []; + server.on("error", (error: Error) => errors.push(error)); + server.on("disconnect", ({ reason }: { reason: Uint8Array }) => reasons.push(Buffer.from(reason).toString("utf8"))); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + try { + connectFromPortZero(serverSocket, connectionId); + // Many resend ticks: nothing the port-0 session owed is ever retried. + await sleep(100); + // The goodbye names the error by its code, never by a message quoting the address. + assert.deepEqual(reasons, ["packet could not be sent: ERR_SOCKET_BAD_PORT"]); + + const frames: string[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(Buffer.from(frame.payload).toString("utf8"))); + client.connect(); + await waitFor(() => client.connected && server.connected, "a real client's handshake"); + client.send("schema", Buffer.from("after", "utf8")); + await waitFor(() => frames.length === 1, "the real client's frame"); + assert.deepEqual(frames, ["after"]); + assert.deepEqual(errors, []); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.close(); + server.close(); + } +}); + +test("an operation server answering a Connect from port 0 starts no session and keeps serving", async () => { + const connectionId = 0x43554c54; + const uncaught = watchUncaught(); + // The server owns its socket; this captures it so a datagram can arrive from port 0. + const createSocket = dgram.createSocket; + let serverSocket: Socket | undefined; + Object.assign(dgram, { + createSocket: (...args: Parameters) => { + serverSocket = createSocket(...args); + return serverSocket; + }, + }); + const server = await startCultNetOperationServer({ + runtimeId: "operations", + handler: (request): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: request.payloadSchema, + payloadEncoding: request.payloadEncoding, + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "operations", + }), + }).finally(() => Object.assign(dgram, { createSocket })); + const raw = await bindUdpSocket(); + const port = Number(new URL(server.endpoint).port); + const accepts: number[] = []; + raw.on("message", () => accepts.push(Date.now())); + try { + connectFromPortZero(serverSocket!, connectionId); + // The resend timer runs every 25 ms; a session that had started would owe its Accept. + await sleep(150); + assert.equal(server.packetsDropped, 1, "the Connect started no session"); + assert.equal(server.sendFailures, 0); + + // A real peer is still answered. + raw.send(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(0)), port, "127.0.0.1"); + await waitFor(() => accepts.length > 0, "the real peer's Accept"); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + raw.close(); + await server.close(); + } +}); + +test("a server transport whose resend can never be sent ends the session and sends nothing more of it", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203094; + const uncaught = watchUncaught(); + const clientPort = udpPort(clientSocket); + // Sends to the client fail as the test says: later through the callback, or at once. + let mode: "healthy" | "lost" | "never" = "healthy"; + const send = serverSocket.send.bind(serverSocket) as (...args: unknown[]) => void; + Object.assign(serverSocket, { + send: (message: Uint8Array, port: number, address: string, callback?: (error: Error | null) => void) => { + if (port !== clientPort || mode === "healthy") { + send(message, port, address, callback); + } else if (mode === "lost") { + process.nextTick(() => callback?.(Object.assign(new Error("lost"), { code: "EHOSTUNREACH" }))); + } else { + throw Object.assign(new Error("never"), { code: "EMSGSIZE" }); + } + }, + }); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + const errors: string[] = []; + const reasons: string[] = []; + server.on("error", (error: NodeJS.ErrnoException) => errors.push(error.code ?? "")); + server.on("disconnect", ({ reason }: { reason: Uint8Array }) => reasons.push(Buffer.from(reason).toString("utf8"))); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); + mode = "lost"; + server.send("schema", Buffer.from("one", "utf8")); + server.send("schema", Buffer.from("two", "utf8")); + await waitFor(() => server.stats.sendFailures >= 4, "both writes lost, and lost again when resent"); + assert.deepEqual(errors, [], "a loss that may pass is counted, never emitted"); + assert.deepEqual(reasons, []); + + // Both writes are due again together; the first can never be sent. + mode = "never"; + await waitFor(() => reasons.length > 0, "the session to end"); + await sleep(50); + assert.deepEqual(reasons, ["packet could not be sent: EMSGSIZE"]); + assert.equal(server.connected, false); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.close(); + server.close(); + } +}); + +// The kernel refuses a datagram to the limited broadcast address from a socket that has +// not asked for broadcast (EACCES): a real failure that may pass once the socket or the +// route changes, so it is a lost datagram. Node reports it only to the send's callback, +// never on the socket's "error" event, so nothing may turn it into an "error" either. +const UNREACHABLE = { host: "255.255.255.255", port: 9 }; + +test("a client transport with no error listener counts a send the network refuses and does not end the process", async () => { + const socket = await new Promise((resolve) => { + const created = dgram.createSocket("udp4"); + created.bind(0, "0.0.0.0", () => resolve(created)); + }); + const uncaught = watchUncaught(); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket, + mode: "client", + remoteHost: UNREACHABLE.host, + remotePort: UNREACHABLE.port, + connectionId: 0x10203095, + resendDelayMs: 10, + resendPollMs: 5, + }); + try { + client.connect(); + // The Connect and its resends, each refused by the kernel. + await waitFor(() => client.stats.sendFailures >= 5, "the refused Connect and its resends"); + assert.equal(client.connected, false); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.close(); + } +}); + +test("an operation call the network refuses rejects with a timeout and does not end the process", async () => { + const uncaught = watchUncaught(); + try { + await assert.rejects( + invokeCultNetOperation( + `rudp://${UNREACHABLE.host}:${UNREACHABLE.port}`, + { + schemaVersion: "cultnet.operation_request.v0", + messageId: "refused", + serviceId: "service", + operation: "operation", + payloadSchema: "payload", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + sourceRuntimeId: "caller", + }, + { runtimeId: "caller", timeoutMs: 400 }, + ), + /CultNet operation connection timed out/, + ); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + } +}); + +// A send's failure can be reported after the generation that sent it ended: the client +// connected again from the same endpoint in the meantime. That failure is owed by nothing; +// it may not end the generation that replaced it or say goodbye to the endpoint. + +/** Makes the next send to `port` fail as can never pass as built, reported `delayMs` later. */ +function failNextSendLate(socket: Socket, port: number, delayMs: number): { arm: () => void } { + let armed = false; + const send = socket.send.bind(socket) as (...args: unknown[]) => void; + Object.assign(socket, { + send: (message: Uint8Array, toPort: number, address: string, callback?: (error: Error | null) => void) => { + if (toPort !== port || !armed) { + send(message, toPort, address, callback); + return; + } + armed = false; + setTimeout(() => callback?.(Object.assign(new Error("late"), { code: "EMSGSIZE" })), delayMs); + }, + }); + return { arm: () => { armed = true; } }; +} + +/** A client driven packet by packet from one socket, so it can connect again from the same endpoint. */ +async function rawClient(serverPort: number, connectionId: number) { + const socket = await bindUdpSocket(); + const received: ReturnType[] = []; + socket.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const toServer = (packet: Parameters[0]) => socket.send(encodeRudpPacket(packet), serverPort, "127.0.0.1"); + const connect = async (initialSequence: number) => { + const session = new CultNetRudpSession({ connectionId, initialSequence }); + received.length = 0; + toServer(session.createConnect(Date.now())); + await waitFor(() => received.some((packet) => packet.packetType === "accept"), "the Accept"); + const accept = received.find((packet) => packet.packetType === "accept")!; + session.receive(accept, Date.now()); + toServer(session.createAckForReceived(accept.sequence)); + return session; + }; + return { socket, received, toServer, connect }; +} + +test("a transport ignores a send failure reported after the generation that sent it ended", async () => { + const serverSocket = await bindUdpSocket(); + const connectionId = 0x10203096; + const uncaught = watchUncaught(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 1_000, + }); + const reasons: string[] = []; + server.on("disconnect", ({ reason }: { reason: Uint8Array }) => reasons.push(Buffer.from(reason).toString("utf8"))); + const client = await rawClient(udpPort(serverSocket), connectionId); + const late = failNextSendLate(serverSocket, udpPort(client.socket), 150); + try { + await client.connect(10); + late.arm(); + server.send("schema", Buffer.from("old", "utf8")); + await client.connect(5_000); + await sleep(250); + + assert.deepEqual(reasons, []); + assert.equal(server.connected, true); + assert.deepEqual(client.received.filter((packet) => packet.packetType === "disconnect"), []); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.socket.close(); + server.close(); + } +}); + +test("an operation server ignores a send failure reported after the generation that sent it ended", async () => { + const connectionId = 0x43554c54; + const createSocket = dgram.createSocket; + let serverSocket: Socket | undefined; + Object.assign(dgram, { + createSocket: (...args: Parameters) => { + serverSocket = createSocket(...args); + return serverSocket; + }, + }); + const server = await startCultNetOperationServer({ + runtimeId: "operations", + handler: (request): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: request.payloadSchema, + payloadEncoding: request.payloadEncoding, + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "operations", + }), + }).finally(() => Object.assign(dgram, { createSocket })); + const client = await rawClient(Number(new URL(server.endpoint).port), connectionId); + const late = failNextSendLate(serverSocket!, udpPort(client.socket), 150); + const request = (session: CultNetRudpSession, messageId: string) => session.sendMany("schema", encode(encodeCultNetMessageForWire({ + schemaVersion: "cultnet.operation_request.v0", + messageId, + serviceId: "service", + operation: "operation", + payloadSchema: "payload", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + sourceRuntimeId: "caller", + } satisfies CultNetOperationRequestMessage, "cultnet.schema.v0")), { reliable: true, ordered: true, nowMs: Date.now() }); + try { + const old = await client.connect(10); + // The response to this request is the next send, and it fails after the client reconnected. + late.arm(); + for (const packet of request(old, "old")) client.toServer(packet); + const renewed = await client.connect(5_000); + await sleep(250); + assert.deepEqual(client.received.filter((packet) => packet.packetType === "disconnect"), []); + assert.equal(server.sendFailures, 0); + + // The session that replaced it is still served. + let read = client.received.length; + let answered = 0; + for (const packet of request(renewed, "after")) client.toServer(packet); + await waitFor(() => { + for (; read < client.received.length; read += 1) { + const packet = client.received[read]!; + if (packet.packetType === "data") answered += renewed.receive(packet, Date.now()).delivered.length; + } + return answered > 0; + }, "the renewed session's response"); + } finally { + client.socket.close(); + await server.close(); + } +}); + +test("a caller's send larger than a UDP datagram throws and takes nothing from the session", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203097; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + }); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + }); + const frames: number[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(frame.payload.byteLength)); + // 36 header bytes and the six of "schema": the largest payload one IPv4 datagram carries. + const largest = 65_507 - 36 - "schema".length; + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); + await waitFor(() => client.outstandingReliablePacketCount === 0, "the Connect acknowledged"); + + client.send("schema", new Uint8Array(largest)); + assert.throws(() => client.send("schema", new Uint8Array(largest + 1)), { code: "EMSGSIZE" }); + assert.equal(client.stats.framesSent, 1); + client.send("schema", Buffer.from("next", "utf8")); + + // The refused send took no sequence: the next frame is not held behind a gap. + await waitFor(() => frames.length === 2, "both sent frames"); + assert.deepEqual(frames, [largest, 4]); + assert.equal(client.connected, true); + await waitFor(() => client.outstandingReliablePacketCount === 0, "both frames acknowledged"); + } finally { + client.close(); + server.close(); + } +}); + +// The pre-check measures one fragment, the largest datagram a fragmented send makes. Fragments +// of 1 KiB and of 40,000 bytes both leave a 70,000-byte send legal; the second size is large +// enough that any small multiple of it would not be. +for (const maxFragmentBytes of [1024, 40_000]) { + test(`a fragmented send larger than one UDP datagram is sent, because each datagram carries one fragment (${maxFragmentBytes}-byte fragments)`, async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203095; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + }); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + maxFragmentBytes, + }); + const frames: number[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(frame.payload.byteLength)); + // Larger than any one datagram can carry, over IPv4 or IPv6. + const payloadBytes = 70_000; + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); + + assert.doesNotThrow(() => client.send("schema", new Uint8Array(payloadBytes))); + assert.equal(client.stats.framesSent, 1); + + await waitFor(() => frames.length === 1, "the reassembled frame"); + assert.deepEqual(frames, [payloadBytes]); + assert.equal(client.connected, true); + } finally { + client.close(); + server.close(); + } + }); +} diff --git a/src/GameCult.Networking/CultNetRudpSchemaServer.cs b/src/GameCult.Networking/CultNetRudpSchemaServer.cs index e19fccc3..de8d0154 100644 --- a/src/GameCult.Networking/CultNetRudpSchemaServer.cs +++ b/src/GameCult.Networking/CultNetRudpSchemaServer.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Net; using System.Net.Sockets; +using System.Text; using System.Threading.Tasks; using GameCult.Logging; @@ -169,6 +170,9 @@ namespace GameCult.Networking /// public CultNetTransportStats Stats => _transport.Stats; + // The listener this server drives, for tests that inject send failures through its seams. + internal CultNetRudpSocketTransportServer Transport => _transport; + /// /// Gets the currently tracked RUDP peers. /// @@ -283,26 +287,47 @@ namespace GameCult.Networking return new RudpCultNetSchemaPollResult(consumed, dispatched); } + /// + /// The goodbye's reason when a handler fails on a peer's message: the server could not handle + /// it, which says nothing about whether the peer's packet was well formed. + /// + internal static readonly byte[] HandlerFailedReason = Encoding.UTF8.GetBytes("server could not handle a message"); + + private static string SessionOutcome(bool ended) => + ended ? "that peer's session ended." : "that peer's session had already ended."; + private async Task DispatchAsync(CultNetRudpSocketServerFrame delivered) { if (!string.Equals(delivered.Frame.ChannelId, "schema", StringComparison.Ordinal)) return false; - var message = CultNetSchemaMessageSerialization.Deserialize(delivered.Frame.Payload); + // A peer's message that cannot be decoded (malformed bytes, an unknown schemaVersion) or + // handled ends that peer's session, never the poll: the drain goes on serving every other + // peer. The transport already acknowledged the frame, so the session cannot be kept (a + // retransmit would never be handled). The peer is told, and the drop is counted and logged. + // The C# TCP schema server (CultNetTcpSchemaTransport.ServePeerAsync) likewise ends the one + // peer whose message fails to decode or handle. A handler is still expected to answer its + // own typed refusals on the wire (HandleSnapshotRequestV1Async does, R-AM); this is the + // backstop for whatever it does not. The log names the fault's type only: its message can + // quote what the peer sent. + ICultNetSchemaMessage message; + try + { + message = CultNetSchemaMessageSerialization.Deserialize(delivered.Frame.Payload); + } + catch (Exception ex) + { + // The peer sent something no runtime can read: the refusal is the peer's. + var ended = _transport.EndPeerSession(delivered.Peer, CultNetRudpSession.RefusedPacketReason); + Logger.LogError($"CultNet RUDP schema server could not decode a peer's message ({ex.GetType().Name}); {SessionOutcome(ended)}"); + return false; + } + if (!_handlers.TryGetValue(message.GetType(), out var handler) || handler == null) { return false; } - // R-AG: a handler is expected to catch its own typed refusals (HandleSnapshotRequestV1Async - // catches the three selection exceptions and answers cursor_invalid/selection_invalid/ - // reference_outside_target on the wire, R-AM) - this catch is the backstop for whatever a - // handler does not, so an untyped exception from a malformed/hostile message can never - // reach the poll loop and stall the server for every other peer. It still answers nothing - // on the wire (an exception this generic carries no typed refusal to report) and drops the - // one malformed dispatch, exactly as an unregistered handler already does just above - - // R-AM: but it always logs. A backstop that swallows silently is a compensator, and an - // untested, unlogged one hid the SM-6 ArgumentNullException regression for a whole batch. try { var result = handler.DynamicInvoke(message, new RudpCultNetSchemaServerPeer(this, delivered.Peer)); @@ -313,11 +338,14 @@ namespace GameCult.Networking } catch (Exception ex) { - // DynamicInvoke wraps a synchronously-thrown handler fault in TargetInvocationException, - // whose own .Message is the generic "Exception has been thrown by the target of an - // invocation." - log the real fault, not that wrapper. + // DynamicInvoke wraps a synchronously-thrown handler fault in TargetInvocationException; + // name the real fault, not that wrapper. The fault is the server's (a response too large + // to send, a handler bug) or a refusal the handler did not type, so the goodbye does not + // say the peer's packet was refused. A response that failed after its first fragment + // already ended the session over the send, and the log says so. var reported = ex is System.Reflection.TargetInvocationException { InnerException: { } inner } ? inner : ex; - Logger.LogError($"CultNet RUDP schema dispatch failed for {message.GetType().Name}: {reported.Message}"); + var ended = _transport.EndPeerSession(delivered.Peer, HandlerFailedReason); + Logger.LogError($"CultNet RUDP schema server's {message.GetType().Name} handler failed ({reported.GetType().Name}); {SessionOutcome(ended)}"); return false; } return true; diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 9731b7f9..a6102de3 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -1,5 +1,6 @@ using System; using System.Buffers.Binary; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.IO; @@ -39,6 +40,12 @@ namespace GameCult.Networking /// the peer, or a packet the session refuses. /// public long PacketsDropped { get; internal set; } + /// + /// Gets the number of datagrams a multi-peer server could not send to one peer. Each is + /// that peer's lost datagram: a reliable packet stays pending and is resent, and no other + /// peer is affected. Zero for single-peer transports, whose callers get the send error. + /// + public long SendFailures { get; internal set; } internal CultNetTransportStats Snapshot() { @@ -48,7 +55,8 @@ namespace GameCult.Networking FramesReceived = FramesReceived, BytesSent = BytesSent, BytesReceived = BytesReceived, - PacketsDropped = PacketsDropped + PacketsDropped = PacketsDropped, + SendFailures = SendFailures }; } } @@ -1454,10 +1462,64 @@ namespace GameCult.Networking /// sequence, so the peer's sequences are forgotten before the goodbye is built: its ack field /// would otherwise acknowledge the very frame the session refused. /// - internal CultNetRudpPacket EndRefused() + internal CultNetRudpPacket EndRefused() => EndWith(RefusedPacketReason); + + /// + /// Ends a session whose peer's frame the application above it refused or failed on, after the + /// session acknowledged it. As with , the peer's sequences are forgotten + /// before the goodbye is built. + /// + internal CultNetRudpPacket EndWith(byte[] reason) { ResetPeerState(); - return CreateDisconnect(RefusedPacketReason); + return CreateDisconnect(reason); + } + + /// + /// Whether a failed send can never succeed for the datagram as built, so resending it is + /// pointless: it is too large for a datagram (EMSGSIZE, WSAEMSGSIZE), its address is + /// malformed for the socket (EINVAL, WSAEINVAL), or belongs to another address family + /// (EAFNOSUPPORT, WSAEAFNOSUPPORT). Everything else (no route, full buffers, firewall drops, + /// a refusal reported for an earlier datagram, an interrupted call) is a property of the path + /// or the moment and may pass, so it is a lost datagram, not a reason to end a session. + /// + internal static bool IsPermanentSendError(SocketException error) => + error.SocketErrorCode == SocketError.MessageSize || + error.SocketErrorCode == SocketError.InvalidArgument || + error.SocketErrorCode == SocketError.AddressFamilyNotSupported; + + internal const string UnsendablePacketReason = "packet could not be sent"; + + /// + /// Ends a session that owes its peer a packet that can never be sent as built. Resending it + /// would fail forever, and dropping it would leave a reliable sequence the peer waits on for + /// good, so the session cannot be kept. The goodbye's reason names the error. + /// + internal CultNetRudpPacket EndUnsendable(SocketException error) + { + ResetPeerState(); + return CreateDisconnect(Encoding.UTF8.GetBytes(UnsendablePacketReason + ": " + error.Message)); + } + + internal uint NextSequence => _nextSequence; + + /// + /// Undoes a send that never reached the wire: every packet it created at or after + /// , pending or queued, leaves no trace and its sequences are + /// reissued, so the peer never sees a gap. Only valid while nothing else has been sent since. + /// + internal void WithdrawSend(uint firstSequence) + { + lock (_pendingReliableGate) + { + foreach (var sequence in _pendingReliable.Keys.Where(sequence => sequence >= firstSequence).ToArray()) + _pendingReliable.Remove(sequence); + var kept = _queuedReliable.Where(packet => packet.Sequence < firstSequence).ToArray(); + _queuedReliable.Clear(); + foreach (var packet in kept) + _queuedReliable.Enqueue(packet); + } + _nextSequence = firstSequence; } /// @@ -2677,6 +2739,11 @@ namespace GameCult.Networking /// Gets the last transport-level remote disconnect reason, if one was received. /// public byte[]? DisconnectReason { get; internal set; } + + // The goodbye's reason once the session ended over a packet it could never send. Written + // under SessionGate by whichever thread's send found the failure; the polling thread reads + // it when it retires the peer. + internal byte[]? UnsendableReason { get; set; } } /// @@ -2714,6 +2781,10 @@ namespace GameCult.Networking private readonly CultNetTransportStats _stats = new CultNetTransportStats(); private readonly Dictionary _peers = new Dictionary(StringComparer.Ordinal); private readonly Queue _deliveredFrames = new Queue(); + // Peers whose session ended over a packet it could never send, waiting for the polling thread + // to retire them. Only the polling thread changes _peers or raises PeerDisconnected; a send on + // a caller's thread ends the session and leaves the peer here. + private readonly ConcurrentQueue _unsendablePeers = new ConcurrentQueue(); private bool _disposed; /// @@ -2775,8 +2846,28 @@ namespace GameCult.Networking public void Send(CultNetRudpSocketServerPeer peer, string channelId, byte[] payload) { if (peer == null) throw new ArgumentNullException(nameof(peer)); + SocketException? unsendable = null; + var failedIndex = 0; + long generation; lock (peer.SessionGate) - SendPackets(peer, peer.Session.SendMany(channelId, payload, ChannelSendOptions(channelId), _maxFragmentBytes)); + { + var firstSequence = peer.Session.NextSequence; + var packets = peer.Session.SendMany(channelId, payload, ChannelSendOptions(channelId), _maxFragmentBytes); + generation = peer.Session.Generation; + unsendable = SendPackets(peer, packets, out failedIndex); + // A caller-directed send that can never succeed is the caller's error and queues + // nothing. Only when earlier fragments are already on the wire has the peer seen + // part of it, and then the session cannot be kept. + if (unsendable != null && failedIndex == 0) + peer.Session.WithdrawSend(firstSequence); + } + if (unsendable != null) + { + // The polling thread retires the peer; this thread only ends its session. + if (failedIndex > 0) + EndUnsendableSession(peer, unsendable, generation); + throw unsendable; + } _stats.FramesSent++; } @@ -2821,6 +2912,7 @@ namespace GameCult.Networking /// public bool TryReceiveOnce(out CultNetRudpSocketServerFrame? delivered) { + RetireUnsendablePeers(); if (_deliveredFrames.Count > 0) { delivered = _deliveredFrames.Dequeue(); @@ -2883,13 +2975,24 @@ namespace GameCult.Networking _peers.TryGetValue(peerKey, out var admitted); // The admitted peer's session decides whether this Connect repeats the one it accepted: // answer a repeat with the Accept already owed and queue nothing. Any other Connect is - // a new generation. + // a new generation, and the admitted peer's session ends here, under the gate every + // Send takes: a caller still holding the replaced peer can send nothing more into the + // endpoint the new session now owns. if (admitted != null) { + var repeats = false; + SocketException? unsendableReply = null; + long admittedGeneration; lock (admitted.SessionGate) { - if (admitted.Session.ConnectRepeats(packet)) + admittedGeneration = admitted.Session.Generation; + if (!admitted.Session.ConnectRepeats(packet)) { + admitted.Session.ResetPeerState(); + } + else + { + repeats = true; CultNetRudpPacket reply; try { @@ -2900,10 +3003,15 @@ namespace GameCult.Networking _stats.PacketsDropped++; return true; } - SendPacket(admitted.RemoteEndPoint, reply); - return true; + unsendableReply = SendPacket(admitted.RemoteEndPoint, reply); } } + if (repeats) + { + if (unsendableReply != null) + EndUnsendablePeer(admitted, unsendableReply, admittedGeneration); + return true; + } } var peer = new CultNetRudpSocketServerPeer( @@ -2916,9 +3024,20 @@ namespace GameCult.Networking ResendDelayMs = _resendDelayMs, MaxPendingReliablePackets = _maxPendingReliablePackets })); - _peers[peerKey] = peer; + SocketException? unsendableAccept; lock (peer.SessionGate) - SendPacket(peer.RemoteEndPoint, peer.Session.AcceptConnect(packet, NowMs(), _acceptPayload)); + unsendableAccept = SendPacket(peer.RemoteEndPoint, peer.Session.AcceptConnect(packet, NowMs(), _acceptPayload)); + if (unsendableAccept != null) + { + // The peer cannot be answered, so no session starts, and the one it replaced has + // already ended. + _peers.Remove(peerKey); + _stats.PacketsDropped++; + } + else + { + _peers[peerKey] = peer; + } if (admitted != null) { admitted.DisconnectReason = ReplacedPeerReason; @@ -2935,6 +3054,8 @@ namespace GameCult.Networking CultNetRudpReceiveResult? outcome; CultNetRudpPacket? acknowledgement = null; + SocketException? unsendable = null; + long existingGeneration = 0; lock (existingPeer.SessionGate) { try @@ -2951,20 +3072,15 @@ namespace GameCult.Networking if (outcome == null) { - try - { - SendPacket(existingPeer.RemoteEndPoint, existingPeer.Session.EndRefused()); - } - catch (SocketException) - { - // Best effort: the session ends whether or not the peer hears it. - } + // The session ends whether or not the peer hears the goodbye. + SendPacket(existingPeer.RemoteEndPoint, existingPeer.Session.EndRefused()); } else { + existingGeneration = existingPeer.Session.Generation; if (outcome.Reply != null) - SendPacket(existingPeer.RemoteEndPoint, outcome.Reply); - SendPackets(existingPeer, outcome.ReadyToSend); + unsendable = SendPacket(existingPeer.RemoteEndPoint, outcome.Reply); + unsendable ??= SendPackets(existingPeer, outcome.ReadyToSend, out _); if (packet.PacketType == CultNetRudpPacketType.Data) acknowledgement = existingPeer.Session.CreateAckForReceived(packet.Sequence); } @@ -3001,8 +3117,10 @@ namespace GameCult.Networking } delivered = _deliveredFrames.Count > 0 ? _deliveredFrames.Dequeue() : null; - if (acknowledgement != null) - SendPacket(existingPeer.RemoteEndPoint, acknowledgement); + if (acknowledgement != null && unsendable == null) + unsendable = SendPacket(existingPeer.RemoteEndPoint, acknowledgement); + if (unsendable != null) + EndUnsendablePeer(existingPeer, unsendable, existingGeneration); return true; } @@ -3012,23 +3130,118 @@ namespace GameCult.Networking /// public void PollResends() { + RetireUnsendablePeers(); if (_socket.Poll(0, SelectMode.SelectRead)) return; foreach (var peer in _peers.Values.ToArray()) { + SocketException? unsendable; + long generation; lock (peer.SessionGate) - SendPackets(peer, peer.Session.DueResends(NowMs())); + { + var due = peer.Session.DueResends(NowMs()); + generation = peer.Session.Generation; + unsendable = SendPackets(peer, due, out _); + } + if (unsendable != null) + EndUnsendablePeer(peer, unsendable, generation); } } - private void SendPackets(CultNetRudpSocketServerPeer peer, IReadOnlyList packets) + // Stops at the first packet that can never be sent as built and returns its error and + // index; a transient failure is a lost datagram and the loop goes on. + private SocketException? SendPackets(CultNetRudpSocketServerPeer peer, IReadOnlyList packets, out int failedIndex) { + failedIndex = 0; for (var index = 0; index < packets.Count; index++) { - SendPacket(peer.RemoteEndPoint, packets[index]); + var unsendable = SendPacket(peer.RemoteEndPoint, packets[index]); + if (unsendable != null) + { + failedIndex = index; + return unsendable; + } if ((index + 1) % WireBurstPackets == 0 && index + 1 < packets.Count) Thread.Yield(); } + return null; + } + + // An admitted peer owes a packet that can never be sent as built. Inside a poll that ends the + // peer's session, never the poll: the peer is told, and PeerDisconnected carries a reason that + // names the error. + private void EndUnsendablePeer(CultNetRudpSocketServerPeer peer, SocketException error, long generation) + { + EndUnsendableSession(peer, error, generation); + RetireUnsendablePeers(); + } + + // Ends the session and tells the peer, on whichever thread found the failure. The peer stays in + // _peers until the polling thread retires it. The failure belongs to the generation the thread + // read under the gate when it sent; if that generation has already ended (another thread + // ended it over its own failure, or a new Connect from the endpoint replaced it), or the + // thread read it only after the session had ended (a Pong or ack answered on an ended + // session the polling thread has not yet retired), there is nothing left to end, and a + // goodbye would reach the session that now owns the endpoint or repeat the first one. The + // goodbye is built and sent under the gate, so no Connect is admitted between the two. + private void EndUnsendableSession(CultNetRudpSocketServerPeer peer, SocketException error, long generation) + { + lock (peer.SessionGate) + { + if (peer.Session.Generation != generation || peer.Session.Ended) + return; + var goodbye = peer.Session.EndUnsendable(error); + peer.UnsendableReason = goodbye.Payload; + SendPacket(peer.RemoteEndPoint, goodbye); + } + _unsendablePeers.Enqueue(peer); + } + + // Polling thread only. Ends the session of a peer whose delivered frame the application could not + // decode or handle, with a goodbye carrying the application's reason. The frame is already + // acknowledged, so the session cannot be kept: a retransmit would never be handled. The peer + // is told, the drop is counted, and the peer's frames still queued are not delivered. A + // session that already ended (a new Connect replaced it, or a send failure ended it) owes no + // goodbye, since the endpoint may belong to a new session; the call then returns false. + internal bool EndPeerSession(CultNetRudpSocketServerPeer peer, byte[] reason) + { + if (peer == null) throw new ArgumentNullException(nameof(peer)); + if (reason == null) throw new ArgumentNullException(nameof(reason)); + _stats.PacketsDropped++; + lock (peer.SessionGate) + { + if (peer.Session.Ended) + return false; + SendPacket(peer.RemoteEndPoint, peer.Session.EndWith(reason)); + } + + var kept = _deliveredFrames.Where(frame => !ReferenceEquals(frame.Peer, peer)).ToArray(); + _deliveredFrames.Clear(); + foreach (var frame in kept) + _deliveredFrames.Enqueue(frame); + + var key = RemoteKey(peer.RemoteEndPoint); + if (!_peers.TryGetValue(key, out var current) || !ReferenceEquals(current, peer)) + return true; + _peers.Remove(key); + peer.DisconnectReason = reason; + PeerDisconnected?.Invoke(peer); + return true; + } + + // Polling thread only. A peer a new Connect already replaced was reported then, and its key now + // names the peer that replaced it. + private void RetireUnsendablePeers() + { + while (_unsendablePeers.TryDequeue(out var peer)) + { + var key = RemoteKey(peer.RemoteEndPoint); + if (!_peers.TryGetValue(key, out var current) || !ReferenceEquals(current, peer)) + continue; + _peers.Remove(key); + peer.DisconnectReason = peer.UnsendableReason; + PeerDisconnected?.Invoke(peer); + } } /// @@ -3043,13 +3256,55 @@ namespace GameCult.Networking _socket.Dispose(); } - private void SendPacket(EndPoint remoteEndPoint, CultNetRudpPacket packet) + // A transient send failure is that peer's lost datagram: counted, never thrown. A reliable + // packet stays pending and is resent; an unreliable one was allowed to be lost. A permanent + // failure (CultNetRudpSession.IsPermanentSendError) is returned: the datagram can never be + // sent as built, and the caller decides between refusing the send and ending the peer's + // session. Only an encode failure, the listener's own bug, throws. + private SocketException? SendPacket(EndPoint remoteEndPoint, CultNetRudpPacket packet) { var wire = CultNetRudpPacketCodec.Encode(packet); - var sent = _socket.SendTo(wire, remoteEndPoint); - _stats.BytesSent += sent; + BeforeSend?.Invoke(remoteEndPoint, packet); + try + { + if (UnsendableAfter.TryGetValue(remoteEndPoint, out var remaining)) + { + if (remaining > 0) + { + UnsendableAfter[remoteEndPoint] = remaining - 1; + } + else + { + UnsendableAfter.Remove(remoteEndPoint); + throw new SocketException((int)SocketError.MessageSize); + } + } + if (FailingSendPeers.Contains(remoteEndPoint)) + throw new SocketException((int)SocketError.HostUnreachable); + var sent = _socket.SendTo(wire, remoteEndPoint); + _stats.BytesSent += sent; + } + catch (SocketException error) when (CultNetRudpSession.IsPermanentSendError(error)) + { + return error; + } + catch (SocketException) + { + _stats.SendFailures++; + } + return null; } + // Test seams, standing in for paths a loopback peer cannot be made to have. Peers whose every + // datagram fails to send transiently: + internal HashSet FailingSendPeers { get; } = new HashSet(); + + // Peers whose next datagram after this many fails permanently, once. + internal Dictionary UnsendableAfter { get; } = new Dictionary(); + + // Runs before each datagram is sent, to order another thread's work between two sends. + internal Action? BeforeSend { get; set; } + private static string RemoteKey(EndPoint endpoint) { return endpoint.ToString() ?? string.Empty; diff --git a/src/GameCult.Networking/GameCult.Networking.csproj b/src/GameCult.Networking/GameCult.Networking.csproj index 1eef5653..08ef2ed6 100644 --- a/src/GameCult.Networking/GameCult.Networking.csproj +++ b/src/GameCult.Networking/GameCult.Networking.csproj @@ -15,6 +15,10 @@ + + + + diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs new file mode 100644 index 00000000..b8de674f --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs @@ -0,0 +1,293 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading.Tasks; +using GameCult.Logging; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// A schema server's poll serves every peer whatever one peer sends. A message that cannot be + /// decoded or handled is that peer's failure: its session ends, the drop is counted and logged, + /// and the drain goes on to the other peers. + /// + public class CultNetRudpSchemaServerPeerInputTests + { + private const uint ConnectionId = 0x10203096; + private const string Canary = "canary-5f1c9e0b"; + + private sealed class Client + { + public Socket Socket = null!; + public CultNetRudpSession Session = null!; + public EndPoint EndPoint => Socket.LocalEndPoint!; + } + + private sealed class CapturingLogger : ILogger + { + public List Errors { get; } = new(); + public void LogInfo(string message) { } + public void LogWarning(string message) { } + public void LogError(string message) => Errors.Add(message); + public void LogDebug(string message) { } + } + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return socket; + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + private static async Task Connect(RudpCultNetSchemaServer server) + { + var client = new Client + { + Socket = Bind(), + Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }) + }; + client.Socket.SendTo(CultNetRudpPacketCodec.Encode(client.Session.CreateConnect(0)), server.LocalEndPoint); + await server.PollAvailableAsync(8); + client.Session.Receive(Drain(client.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + return client; + } + + private static void SendSchema(RudpCultNetSchemaServer server, Client client, byte[] payload) => + client.Socket.SendTo( + CultNetRudpPacketCodec.Encode(client.Session.Send("schema", payload, new CultNetRudpSendOptions { Reliable = true, Ordered = true })), + server.LocalEndPoint); + + private static byte[] Malformed() => new byte[] { 0xc1 }; + + // Well-formed MessagePack naming a schema version no runtime knows. The decoder's own error + // quotes that version. + private static byte[] UnknownSchemaVersion() => + MessagePackSerializer.Serialize( + new Dictionary { ["schemaVersion"] = "cultnet." + Canary + ".v0", ["messageId"] = Canary }, + CultNetSchemaMessageSerialization.Options); + + private sealed class Scene : IDisposable + { + public RudpCultNetSchemaServer Server = null!; + public Client Bad = null!; + public Client Good = null!; + public CapturingLogger Logger = new(); + public List<(string MessageId, EndPoint From)> Served = new(); + public List Ended = new(); + + public void Dispose() + { + Server.Dispose(); + Bad.Socket.Dispose(); + Good.Socket.Dispose(); + } + } + + // The bad peer's frame is read first, so the drain reaches the good peer only by getting past it. + private static async Task BadThenGood(byte[] badPayload) + { + var scene = new Scene + { + Server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-peer-input", + ConnectionId = ConnectionId + }) + }; + scene.Server.Logger = scene.Logger; + scene.Server.OnCultNet((message, peer) => + scene.Served.Add((message.MessageId, peer.RemoteEndPoint))); + scene.Server.PeerDisconnected += peer => scene.Ended.Add(((RudpCultNetSchemaServerPeer)peer).RemoteEndPoint); + scene.Bad = await Connect(scene.Server); + scene.Good = await Connect(scene.Server); + + SendSchema(scene.Server, scene.Bad, badPayload); + SendSchema(scene.Server, scene.Good, CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "good-request" })); + return scene; + } + + [Test] + public async Task AMalformedPayloadDoesNotStopTheDrainServingAnotherPeer() + { + using var scene = await BadThenGood(Malformed()); + + Assert.DoesNotThrowAsync(async () => await scene.Server.PollAvailableAsync(16)); + + Assert.That(scene.Served, Is.EqualTo(new[] { ("good-request", scene.Good.EndPoint) })); + } + + [Test] + public async Task AnUnknownSchemaVersionDoesNotStopTheDrainServingAnotherPeer() + { + using var scene = await BadThenGood(UnknownSchemaVersion()); + + Assert.DoesNotThrowAsync(async () => await scene.Server.PollAvailableAsync(16)); + + Assert.That(scene.Served, Is.EqualTo(new[] { ("good-request", scene.Good.EndPoint) })); + } + + [Test] + public async Task PollOnceDoesNotThrowForAPeersUndecodableMessage() + { + using var scene = await BadThenGood(Malformed()); + + Assert.DoesNotThrowAsync(async () => + { + for (var attempt = 0; attempt < 16; attempt++) + await scene.Server.PollOnceAsync(); + }); + + Assert.That(scene.Served, Is.EqualTo(new[] { ("good-request", scene.Good.EndPoint) })); + } + + [Test] + public async Task APeerWhoseMessageCannotBeDecodedHasItsSessionEnded() + { + using var scene = await BadThenGood(Malformed()); + var dropped = scene.Server.Stats.PacketsDropped; + + try { await scene.Server.PollAvailableAsync(16); } + catch (MessagePackSerializationException) { Assert.Fail("the poll threw for a peer's input"); } + + // The frame was already acknowledged, so the session cannot be kept: the peer is told, + // the listener forgets it, and the drop is counted. + Assert.That(scene.Ended, Is.EqualTo(new[] { scene.Bad.EndPoint })); + Assert.That(scene.Server.Peers.Select(peer => peer.RemoteEndPoint), Is.EqualTo(new[] { scene.Good.EndPoint })); + Assert.That(scene.Server.Stats.PacketsDropped, Is.EqualTo(dropped + 1)); + var goodbye = Drain(scene.Bad.Socket).Single(packet => packet.PacketType == CultNetRudpPacketType.Disconnect); + Assert.That(scene.Bad.Session.Receive(goodbye, 1).Disconnected, Is.True); + Assert.That(Drain(scene.Good.Socket).Select(packet => packet.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(scene.Logger.Errors, Has.Count.EqualTo(1)); + } + + // The peer's malformed message and a well-formed one after it arrive in one datagram's + // delivery: the second is held for order, and both are released together. The session ends + // on the first, so the second belongs to no session and is never handled. + [Test] + public async Task ARefusedPeersFramesStillQueuedAreNotHandled() + { + var logger = new CapturingLogger(); + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-peer-input", + ConnectionId = ConnectionId + }); + server.Logger = logger; + var served = new List(); + server.OnCultNet((message, _) => served.Add(message.MessageId)); + var bad = await Connect(server); + var options = new CultNetRudpSendOptions { Reliable = true, Ordered = true }; + var malformed = bad.Session.Send("schema", Malformed(), options); + var after = bad.Session.Send("schema", CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "after-the-refused-one" }), options); + bad.Socket.SendTo(CultNetRudpPacketCodec.Encode(after), server.LocalEndPoint); + bad.Socket.SendTo(CultNetRudpPacketCodec.Encode(malformed), server.LocalEndPoint); + + await server.PollAvailableAsync(16); + + Assert.That(served, Is.Empty); + Assert.That(server.Peers, Is.Empty); + Assert.That(logger.Errors, Has.Count.EqualTo(1)); + bad.Socket.Dispose(); + } + + private static async Task<(RudpCultNetSchemaServer Server, Client Client, CapturingLogger Logger)> ServerWhoseHandlerAnswers( + byte[] response, int? maxFragmentBytes, int? unsendableAfter = null) + { + var logger = new CapturingLogger(); + var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-peer-input", + ConnectionId = ConnectionId, + MaxFragmentBytes = maxFragmentBytes + }); + server.Logger = logger; + server.OnCultNet((_, peer) => + { + if (unsendableAfter is { } count) + server.Transport.UnsendableAfter[peer.RemoteEndPoint] = count; + server.Transport.SendSchema(peer.TransportPeer, response); + }); + var client = await Connect(server); + SendSchema(server, client, CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "ask" })); + return (server, client, logger); + } + + // A handler whose own response can never be sent (larger than a datagram, unfragmented) fails + // on the server's account, not the peer's. The session still ends, but its goodbye does not say + // the peer's packet was refused. + [Test] + public async Task AHandlerFaultEndsTheSessionWithAReasonThatDoesNotBlameThePeer() + { + var (server, client, logger) = await ServerWhoseHandlerAnswers(new byte[70_000], maxFragmentBytes: null); + using (server) + { + await server.PollAvailableAsync(16); + + var goodbye = Drain(client.Socket).Single(packet => packet.PacketType == CultNetRudpPacketType.Disconnect); + Assert.That(Encoding.UTF8.GetString(goodbye.Payload!), Is.EqualTo("server could not handle a message")); + Assert.That(server.Peers, Is.Empty); + Assert.That(logger.Errors, Has.Count.EqualTo(1)); + Assert.That(logger.Errors[0], Does.Contain("handler failed (SocketException)")); + Assert.That(logger.Errors[0], Does.Contain("that peer's session ended.")); + } + client.Socket.Dispose(); + } + + // A response that fails after its first fragment has already ended the session over the send, + // with a goodbye that names the send failure. The backstop ends nothing more and says so. + [Test] + public async Task AHandlerWhoseResponseEndedTheSessionIsLoggedAsAlreadyEnded() + { + var (server, client, logger) = await ServerWhoseHandlerAnswers(new byte[3000], maxFragmentBytes: 1000, unsendableAfter: 1); + using (server) + { + await server.PollAvailableAsync(16); + await server.PollAvailableAsync(16); + + var goodbyes = Drain(client.Socket).Where(packet => packet.PacketType == CultNetRudpPacketType.Disconnect).ToList(); + Assert.That(goodbyes, Has.Count.EqualTo(1)); + Assert.That(Encoding.UTF8.GetString(goodbyes[0].Payload!), Does.StartWith("packet could not be sent: ")); + Assert.That(server.Peers, Is.Empty); + Assert.That(logger.Errors, Has.Count.EqualTo(1)); + Assert.That(logger.Errors[0], Does.Contain("that peer's session had already ended.")); + } + client.Socket.Dispose(); + } + + [Test] + public async Task TheDropIsLoggedByErrorTypeWithoutEchoingWhatThePeerSent() + { + using var scene = await BadThenGood(UnknownSchemaVersion()); + + try { await scene.Server.PollAvailableAsync(16); } + catch (MessagePackSerializationException) { Assert.Fail("the poll threw for a peer's input"); } + + // The decoder's own error message quotes the unknown version, which is the peer's input. + Assert.That(scene.Logger.Errors, Has.Count.EqualTo(1)); + Assert.That(scene.Logger.Errors[0], Does.Contain(nameof(MessagePackSerializationException))); + Assert.That(scene.Logger.Errors[0], Does.Not.Contain(Canary)); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs new file mode 100644 index 00000000..81d255f7 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -0,0 +1,646 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// One peer whose datagrams cannot be sent must not stop the listener serving the others. The + /// failure is injected at the listener's one send function through FailingSendPeers. + /// + public class CultNetRudpSendFailureTests + { + private const uint ConnectionId = 0x10203099; + + private sealed class Peer + { + public Socket Socket = null!; + public CultNetRudpSession Session = null!; + public CultNetRudpSocketServerPeer Server = null!; + public EndPoint EndPoint => Socket.LocalEndPoint!; + } + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return socket; + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + private static Peer Connect(CultNetRudpSocketTransportServer server, EndPoint serverEndPoint) + { + var peer = new Peer + { + Socket = Bind(), + Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }) + }; + peer.Socket.SendTo(CultNetRudpPacketCodec.Encode(peer.Session.CreateConnect(0)), serverEndPoint); + server.ReceiveOnce(); + peer.Session.Receive(Drain(peer.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + peer.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(peer.EndPoint)); + return peer; + } + + private static (CultNetRudpSocketTransportServer Server, EndPoint ServerEndPoint, Peer X, Peer Y) TwoPeers(int? maxFragmentBytes = null, uint? initialSequence = null) + { + var serverSocket = Bind(); + var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-send-failure", + Socket = serverSocket, + ConnectionId = ConnectionId, + ResendDelayMs = 5, + MaxFragmentBytes = maxFragmentBytes, + InitialSequence = initialSequence + }); + var x = Connect(server, serverSocket.LocalEndPoint!); + var y = Connect(server, serverSocket.LocalEndPoint!); + return (server, serverSocket.LocalEndPoint!, x, y); + } + + private static CultNetRudpPacket ReliableFrame(Peer peer, string text) => + peer.Session.Send("schema", Encoding.UTF8.GetBytes(text), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); + + [Test] + public void AFailedResendToOnePeerDoesNotStopTheOthersResends() + { + var (server, _, x, y) = TwoPeers(); + using (server) + { + // Both peers' first datagram is lost, so both have a reliable resend due. + server.FailingSendPeers.Add(x.EndPoint); + server.FailingSendPeers.Add(y.EndPoint); + server.SendSchema(x.Server, "for-x"); + server.SendSchema(y.Server, "for-y"); + Assert.That(server.Stats.SendFailures, Is.EqualTo(2)); + + // Y's path recovers; X's stays dead. X is visited first. + server.FailingSendPeers.Remove(y.EndPoint); + Thread.Sleep(30); + Assert.DoesNotThrow(() => server.PollResends()); + + var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList(); + Assert.That(resent, Has.Count.EqualTo(1)); + Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y")); + Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(3)); + Assert.That(server.Peers, Has.Count.EqualTo(2), "a send failure ends no session"); + } + } + + [Test] + public void AFailedAckToOnePeerDoesNotStopTheListenerServingTheOthers() + { + var (server, listenerEndPoint, x, y) = TwoPeers(); + using (server) + { + server.FailingSendPeers.Add(x.EndPoint); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint); + y.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(y, "from-y")), listenerEndPoint); + + var frames = new List(); + for (var attempt = 0; attempt < 20 && frames.Count < 2; attempt++) + { + Assert.DoesNotThrow(() => + { + if (server.ReceiveOnce() is { } frame) + frames.Add(Encoding.UTF8.GetString(frame.Frame.Payload)); + }); + } + + Assert.That(frames, Is.EquivalentTo(new[] { "from-x", "from-y" })); + Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(1), "X's ack was counted as lost"); + Assert.That(Drain(y.Socket).Select(p => p.PacketType), Does.Contain(CultNetRudpPacketType.Ack)); + Assert.That(server.Peers, Has.Count.EqualTo(2)); + } + } + + // A permanent send failure is one that can never succeed for the datagram as built. A + // caller-directed send throws it and queues nothing; inside a poll it ends that peer's + // session and never the poll. + + private static bool IsUnsendableReason(byte[]? reason) + { + var text = Encoding.UTF8.GetString(reason ?? Array.Empty()); + return text.StartsWith("packet could not be sent: ", StringComparison.Ordinal) + && text.Length > "packet could not be sent: ".Length; + } + + private static List RecordDisconnects(CultNetRudpSocketTransportServer server) + { + var ended = new List(); + server.PeerDisconnected += ended.Add; + return ended; + } + + [Test] + public void AnOversizedSendThrowsTheErrorAndQueuesNothing() + { + var (server, _, x, y) = TwoPeers(); + using (server) + { + // Each peer's own Accept is still awaiting its ack; the refused send adds nothing to it. + var outstanding = x.Server.Session.OutstandingReliablePacketCount; + var otherOutstanding = y.Server.Session.OutstandingReliablePacketCount; + var error = Assert.Throws(() => server.SendSchema(x.Server, new byte[70_000]))!; + Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True, error.SocketErrorCode.ToString()); + Assert.That(server.Stats.SendFailures, Is.Zero); + Assert.That(server.Stats.FramesSent, Is.Zero); + Assert.That(server.Peers, Has.Count.EqualTo(2), "the caller's error ends no session"); + Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing is pending or queued to resend"); + + // The refused send consumed no sequence: the next frame is delivered in order instead + // of waiting behind a gap that never fills. + server.SendSchema(x.Server, "next"); + var data = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Data); + var delivered = x.Session.Receive(data, 1).Delivered; + Assert.That(delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)), Is.EqualTo(new[] { "next" })); + Assert.That(y.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(otherOutstanding)); + } + } + + [Test] + public void ARefusedSendWithdrawsTheFragmentsTheWindowQueuedAsWell() + { + var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1); + using (server) + { + var outstanding = x.Server.Session.OutstandingReliablePacketCount; + server.UnsendableAfter[x.EndPoint] = 0; + + // 1,100 one-byte fragments: the flow window admits the first ones and queues the rest, + // and the first datagram can never be sent. + Assert.Throws(() => server.SendSchema(x.Server, new byte[1_100])); + + Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing of the refused send stays queued"); + } + } + + [Test] + public void ASendThatFailsPermanentlyAfterAFragmentLeftEndsTheSession() + { + var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var ended = RecordDisconnects(server); + server.UnsendableAfter[x.EndPoint] = 1; + + var error = Assert.Throws(() => server.SendSchema(x.Server, new byte[3000]))!; + Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True); + // The send ends the session and tells the peer; it leaves the peer for the polling + // thread to retire, because only that thread changes the peer set. + Assert.That(x.Server.Connected, Is.False); + Assert.That(Drain(x.Socket).Last().PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(ended, Is.Empty); + + server.ReceiveOnce(); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + } + } + + [Test] + public void APeerEndedByASendOnAnotherThreadIsRetiredByThePollingThread() + { + var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var retiredOn = new List(); + server.PeerDisconnected += _ => retiredOn.Add(Environment.CurrentManagedThreadId); + server.UnsendableAfter[x.EndPoint] = 1; + + SocketException? thrown = null; + var sender = new Thread(() => + { + try { server.SendSchema(x.Server, new byte[3000]); } + catch (SocketException error) { thrown = error; } + }); + sender.Start(); + sender.Join(); + Assert.That(thrown, Is.Not.Null); + Assert.That(server.Peers, Has.Count.EqualTo(2), "the sending thread leaves the peer set alone"); + + server.PollResends(); + Assert.That(retiredOn, Is.EqualTo(new[] { Environment.CurrentManagedThreadId })); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + } + } + + // A client that connects again from the same endpoint replaces the peer the listener held. A + // caller still holding the replaced peer (a handler still running for it, say) reaches nothing + // of the session that replaced it: no data and no goodbye. + + /// X connects again as a new session, and the handshake completes. + private static (CultNetRudpSocketServerPeer Stale, CultNetRudpSocketServerPeer Replacing) Reconnect( + CultNetRudpSocketTransportServer server, EndPoint listenerEndPoint, Peer x) + { + var stale = x.Server; + Drain(x.Socket); + x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint); + server.ReceiveOnce(); + x.Session.Receive(Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + x.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint)); + Assert.That(x.Server, Is.Not.SameAs(stale)); + Assert.That(x.Session.Connected, Is.True); + return (stale, x.Server); + } + + [Test] + public void ASendOnAReplacedPeerThrowsAndNoGoodbyeReachesTheSessionThatReplacedIt() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var (stale, replacing) = Reconnect(server, listenerEndPoint, x); + var ended = RecordDisconnects(server); + Assert.That(stale.Connected, Is.False, "the replaced peer's session ended with the replacement"); + + // Had the stale send gone out, a failure after its first fragment would end the + // session that sent it and say goodbye to the endpoint. + server.UnsendableAfter[x.EndPoint] = 1; + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + server.ReceiveOnce(); + + Assert.That(Drain(x.Socket), Is.Empty, "nothing of the stale send reached the client"); + Assert.That(x.Session.Connected, Is.True); + Assert.That(replacing.Connected, Is.True); + Assert.That(server.Peers, Has.Member(replacing)); + Assert.That(server.Peers, Has.Count.EqualTo(2)); + Assert.That(ended, Is.Empty); + } + } + + // With a fixed initial sequence the stale frame would take the sequence the real one gets, and + // the real one would be dropped as a duplicate. With a drawn one it would land far ahead of + // the new session's window, and nothing after it would be delivered. + [TestCase(1000u)] + [TestCase(null)] + public void ASendOnAReplacedPeerDeliversNothingAndTheReplacingPeersFrameArrives(uint? initialSequence) + { + var (server, listenerEndPoint, x, _) = TwoPeers(initialSequence: initialSequence); + using (server) + { + var (stale, replacing) = Reconnect(server, listenerEndPoint, x); + + Assert.Throws(() => server.SendSchema(stale, "STALE")); + server.SendSchema(replacing, "REAL"); + + var delivered = Drain(x.Socket) + .SelectMany(packet => x.Session.Receive(packet, 1).Delivered) + .Select(frame => Encoding.UTF8.GetString(frame.Payload)) + .ToList(); + Assert.That(delivered, Is.EqualTo(new[] { "REAL" })); + } + } + + // A send that fails after its first fragment ends the generation it began in, and only that + // one. Here X connects again while the send is under way, and the polling thread admits the + // new Connect before the fragment that can never be sent. The hook runs that poll inside the + // send so the order is fixed; in production it runs on the polling thread between the send + // and the failure's handling. The goodbye belonged to the ended generation, and the client + // must not hear it: it would believe itself disconnected while the listener holds it + // connected. + [Test] + public void ASendFailureAfterANewConnectReplacedThePeerSaysNoGoodbyeToTheNewSession() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var stale = x.Server; + Drain(x.Socket); + x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint); + + var fragments = 0; + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Data || ++fragments < 2) + return; + server.BeforeSend = null; + server.ReceiveOnce(); + server.UnsendableAfter[x.EndPoint] = 0; + }; + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + server.ReceiveOnce(); + + var packets = Drain(x.Socket); + Assert.That(packets.Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect), + "the ended generation's goodbye reached the session that replaced it"); + x.Session.Receive(packets.Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + Assert.That(x.Session.Connected, Is.True); + var replacing = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint)); + Assert.That(replacing, Is.Not.SameAs(stale)); + Assert.That(replacing.Connected, Is.True); + } + } + + // A caller's send ends X's session over a fragment it could never send. Before the listener + // retires X, the polling thread answers X's Ping on the ended session, and that Pong can never + // be sent either. The polling thread read the generation only after the session ended, so the + // generation alone cannot tell it the session is gone. It says no second goodbye, and the + // first failure's reason stands. The hook admits the Ping while the first goodbye is on its way, + // after the session ended and before the peer is queued for retirement. + [Test] + public void AFailureAnsweringAPingOnAnAlreadyEndedSessionSaysNoSecondGoodbye() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var ended = RecordDisconnects(server); + byte[]? firstReason = null; + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Disconnect) + return; + server.BeforeSend = null; + firstReason = x.Server.UnsendableReason; + server.UnsendableAfter[x.EndPoint] = 0; + server.ReceiveOnce(); + }; + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreatePing(Encoding.UTF8.GetBytes("ping"))), listenerEndPoint); + server.UnsendableAfter[x.EndPoint] = 1; + + Assert.Throws(() => server.SendSchema(x.Server, new byte[3000])); + server.ReceiveOnce(); + + var goodbyes = Drain(x.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Disconnect).ToList(); + Assert.That(goodbyes, Has.Count.EqualTo(1)); + Assert.That(firstReason, Is.Not.Null); + Assert.That(x.Server.UnsendableReason, Is.SameAs(firstReason), "the second failure overwrote the first's reason"); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(x.Server.DisconnectReason, Is.SameAs(firstReason)); + } + } + + // The goodbye for a failed send is sent under the session's gate. The polling thread, admitting + // a new Connect from X, waits on that gate, so the ended generation's goodbye can never follow + // the new session's Accept. The hook starts the polling thread just before the goodbye leaves + // and gives it a second to get ahead if nothing holds it back. + [Test] + public void AFailedSendsGoodbyeLeavesBeforeANewConnectFromTheEndpointIsAdmitted() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var stale = x.Server; + Drain(x.Socket); + x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint); + + Thread? poller = null; + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Disconnect) + return; + server.BeforeSend = null; + poller = new Thread(() => server.ReceiveOnce()); + poller.Start(); + poller.Join(1000); + }; + server.UnsendableAfter[x.EndPoint] = 1; + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + Assert.That(poller, Is.Not.Null); + poller!.Join(); + + var types = Drain(x.Socket).Select(p => p.PacketType).ToList(); + Assert.That(types, Does.Contain(CultNetRudpPacketType.Disconnect)); + Assert.That(types, Does.Contain(CultNetRudpPacketType.Accept)); + Assert.That(types.IndexOf(CultNetRudpPacketType.Disconnect), Is.LessThan(types.IndexOf(CultNetRudpPacketType.Accept)), + "the ended generation's goodbye followed the new session's Accept"); + } + } + + // An application ending a peer's session (a message it could not decode or handle) says no + // goodbye to a session that already ended: a new Connect replaced it, and the endpoint now + // belongs to the session that replaced it. + [Test] + public void EndingAReplacedPeersSessionSaysNoGoodbyeToTheSessionThatReplacedIt() + { + var (server, listenerEndPoint, x, _) = TwoPeers(); + using (server) + { + var (stale, replacing) = Reconnect(server, listenerEndPoint, x); + var ended = RecordDisconnects(server); + + Assert.That(server.EndPeerSession(stale, Encoding.UTF8.GetBytes("application reason")), Is.False); + + Assert.That(Drain(x.Socket).Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(x.Session.Connected, Is.True); + Assert.That(replacing.Connected, Is.True); + Assert.That(server.Peers, Has.Member(replacing)); + Assert.That(ended, Is.Empty); + } + } + + [Test] + public void ASessionEndedByACallerAndThePollingThreadTogetherSaysGoodbyeOnce() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var ended = RecordDisconnects(server); + // The polling thread acknowledges X's frame outside the session's gate. Just before + // that ack goes out, a caller's send to X fails after its first fragment and ends the + // session; then the ack can never be sent either. + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Ack) + return; + server.BeforeSend = null; + server.UnsendableAfter[x.EndPoint] = 1; + var caller = new Thread(() => + { + try { server.SendSchema(x.Server, new byte[3000]); } + catch (SocketException) { } + }); + caller.Start(); + caller.Join(); + server.UnsendableAfter[x.EndPoint] = 0; + }; + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint); + server.ReceiveOnce(); + server.PollResends(); + + var goodbyes = Drain(x.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Disconnect).ToList(); + Assert.That(goodbyes, Has.Count.EqualTo(1)); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + } + } + + [Test] + public void APermanentFailureInResendsEndsOnlyThatPeersSession() + { + var (server, _, x, y) = TwoPeers(); + using (server) + { + var ended = RecordDisconnects(server); + server.FailingSendPeers.Add(x.EndPoint); + server.FailingSendPeers.Add(y.EndPoint); + server.SendSchema(x.Server, "for-x"); + server.SendSchema(y.Server, "for-y"); + + // X's datagram can now never be sent as built; Y's path recovers. X is visited first. + server.FailingSendPeers.Clear(); + server.UnsendableAfter[x.EndPoint] = 0; + Thread.Sleep(30); + Assert.DoesNotThrow(() => server.PollResends()); + + Assert.That(server.Peers, Is.EqualTo(new[] { y.Server })); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList(); + Assert.That(resent, Has.Count.EqualTo(1)); + Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y")); + var goodbye = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Disconnect); + Assert.That(IsUnsendableReason(goodbye.Payload), Is.True); + } + } + + [Test] + public void APermanentFailureAcknowledgingAFrameEndsTheSessionAfterTheFrameIsDelivered() + { + var (server, listenerEndPoint, x, _) = TwoPeers(); + using (server) + { + var ended = RecordDisconnects(server); + server.UnsendableAfter[x.EndPoint] = 0; + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint); + + CultNetRudpSocketServerFrame? frame = null; + Assert.DoesNotThrow(() => frame = server.ReceiveOnce()); + + Assert.That(frame, Is.Not.Null, "the frame is delivered before the session ends"); + Assert.That(Encoding.UTF8.GetString(frame!.Frame.Payload), Is.EqualTo("from-x")); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + Assert.That(server.Stats.SendFailures, Is.Zero); + } + } + + [Test] + public void APermanentFailureAnsweringARepeatedConnectEndsTheSession() + { + var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-send-failure", + Socket = serverSocket, + ConnectionId = ConnectionId, + ResendDelayMs = 5 + }); + var ended = RecordDisconnects(server); + var peer = Bind(); + var connect = CultNetRudpPacketCodec.Encode( + new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0)); + peer.SendTo(connect, serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + var admitted = server.Peers.Single(); + + server.UnsendableAfter[peer.LocalEndPoint!] = 0; + peer.SendTo(connect, serverSocket.LocalEndPoint!); + Assert.DoesNotThrow(() => server.ReceiveOnce()); + + Assert.That(server.Peers, Is.Empty); + Assert.That(ended, Is.EqualTo(new[] { admitted })); + Assert.That(IsUnsendableReason(admitted.DisconnectReason), Is.True); + } + + [Test] + public void AConnectWhoseAcceptCanNeverBeSentStartsNoSession() + { + var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-send-failure", + Socket = serverSocket, + ConnectionId = ConnectionId, + ResendDelayMs = 5 + }); + var peer = Bind(); + server.UnsendableAfter[peer.LocalEndPoint!] = 0; + peer.SendTo( + CultNetRudpPacketCodec.Encode(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0)), + serverSocket.LocalEndPoint!); + + Assert.DoesNotThrow(() => server.ReceiveOnce()); + + Assert.That(server.Peers, Is.Empty); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(1)); + } + + [Test] + public void APeerEndedInTheResendLoopIsSentNothingMoreFromIt() + { + var (server, _, x, _) = TwoPeers(); + using (server) + { + server.FailingSendPeers.Add(x.EndPoint); + server.SendSchema(x.Server, "one"); + server.SendSchema(x.Server, "two"); + Assert.That(server.Stats.SendFailures, Is.EqualTo(2)); + + server.UnsendableAfter[x.EndPoint] = 0; + Thread.Sleep(30); + server.PollResends(); + + // Both resends are due. The first can never be sent and ends the session; the + // goodbye is the only other datagram attempted at X. + Assert.That(server.Stats.SendFailures, Is.EqualTo(3)); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + } + } + + [Test] + public void AHealthySendCountsItsBytesAndNoFailure() + { + var (server, _, x, _) = TwoPeers(); + using (server) + { + var before = server.Stats.BytesSent; + server.SendSchema(x.Server, "hello"); + var received = Drain(x.Socket).Sum(p => CultNetRudpPacketCodec.Encode(p).Length); + Assert.That(received, Is.GreaterThan("hello".Length)); + Assert.That(server.Stats.BytesSent - before, Is.EqualTo(received)); + Assert.That(server.Stats.SendFailures, Is.Zero); + } + } + + [Test] + public void OnlyADatagramThatCanNeverBeSentIsAPermanentFailure() + { + foreach (var permanent in new[] { SocketError.MessageSize, SocketError.InvalidArgument, SocketError.AddressFamilyNotSupported }) + Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)permanent)), Is.True, permanent.ToString()); + foreach (var transient in new[] + { + SocketError.HostUnreachable, SocketError.NetworkUnreachable, SocketError.NoBufferSpaceAvailable, + SocketError.WouldBlock, SocketError.ConnectionReset, SocketError.ConnectionRefused, + SocketError.AccessDenied, SocketError.NetworkDown, SocketError.AddressNotAvailable, + SocketError.Interrupted, SocketError.TimedOut, SocketError.Shutdown + }) + Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)transient)), Is.False, transient.ToString()); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs index 9b6959c0..e4c31d64 100644 --- a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs @@ -796,8 +796,11 @@ namespace GameCult.Networking.Tests serverThread.Join(); } - Assert.That(logger.Errors, Has.Some.Contains("dispatch-backstop-probe"), - "the backstop must log the handler's own exception message, not swallow it silently"); + // The log names the fault's type. The fault's own message can quote what the peer sent, + // so it is never logged. + Assert.That(logger.Errors, Has.Some.Contains(nameof(InvalidOperationException)), + "the backstop must log the handler's fault, not swallow it silently"); + Assert.That(logger.Errors, Has.None.Contains("dispatch-backstop-probe")); } private sealed class CapturingLogger : ILogger