From def93c5d894f20779635636e74f1b4bb730ee819 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:47:53 +0200 Subject: [PATCH 01/33] Ack Cut D (Rust): a failed send to one peer is that peer's lost datagram The cultmesh-rs document server and the cultnet-rs hub returned the first failed send_to from maintain, poll_once, poll_resends and receive_event_once, so one peer whose datagrams cannot be sent failed every poll for every peer. send_packet now counts a send failure (send_failures() on the document server, CultNetTransportStats.send_failures on the hub) and returns; only an encode failure stays an error. The document server treats recv_from ConnectionReset as Idle, as the socket transport does. Tests inject the failure at the one send function through a cfg(test) set of failing peers and prove the other peer is still served. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 43 +++- .../send_failure_tests.rs | 211 ++++++++++++++++++ packages/cultnet-rs/src/rudp.rs | 27 ++- .../cultnet-rs/src/rudp/send_failure_tests.rs | 124 ++++++++++ packages/cultnet-rs/src/transport.rs | 5 + 5 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs create mode 100644 packages/cultnet-rs/src/rudp/send_failure_tests.rs diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index b99c7145..9ce04b58 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -6,6 +6,8 @@ use cultnet_rs::{ encode_rudp_packet, }; use std::collections::BTreeMap; +#[cfg(test)] +use std::collections::BTreeSet; use std::io::ErrorKind; use std::net::{SocketAddr, UdpSocket}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -208,6 +210,11 @@ pub struct CultMeshRudpDocumentServer { clock: C, options: CultMeshRudpDocumentServerOptions, packets_dropped: u64, + send_failures: u64, + /// Peers whose every datagram fails to send, standing in for an unroutable + /// or full path that a loopback peer cannot be made to have. + #[cfg(test)] + failing_peers: BTreeSet, } impl CultMeshRudpDocumentServer @@ -236,6 +243,9 @@ where clock, options, packets_dropped: 0, + send_failures: 0, + #[cfg(test)] + failing_peers: BTreeSet::new(), }) } @@ -247,6 +257,15 @@ where self.packets_dropped } + /// Datagrams that could not be sent to a peer. Each is that peer's lost + /// datagram: a reliable packet stays pending and is resent, and the peer's + /// session ends by idle timeout, lifetime, refusal or Disconnect, never by + /// the failure. Never an error: one unreachable peer must not stop the + /// poll that serves the others. + pub fn send_failures(&self) -> u64 { + self.send_failures + } + pub fn local_addr(&self) -> Result { Ok(self.socket.local_addr()?) } @@ -265,7 +284,14 @@ where let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES]; let (received, remote_addr) = match self.socket.recv_from(&mut wire) { Ok(value) => value, - Err(error) if error.kind() == ErrorKind::WouldBlock => { + // Windows reports an earlier ICMP port-unreachable on the next + // receive; it names no datagram of this poll. + Err(error) + if matches!( + error.kind(), + ErrorKind::WouldBlock | ErrorKind::ConnectionReset + ) => + { return Ok(CultMeshRudpPollOutcome::Idle); } Err(error) => return Err(error.into()), @@ -627,11 +653,24 @@ where fn send_packet(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { let wire = encode_rudp_packet(packet)?; - self.socket.send_to(&wire, remote_addr)?; + #[cfg(test)] + let sent = if self.failing_peers.contains(&remote_addr) { + Err(std::io::Error::other("injected send failure")) + } else { + self.socket.send_to(&wire, remote_addr) + }; + #[cfg(not(test))] + let sent = self.socket.send_to(&wire, remote_addr); + if sent.is_err() { + self.send_failures += 1; + } Ok(()) } } +#[cfg(test)] +mod send_failure_tests; + fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { if options.max_sessions == 0 { return Err(anyhow!("max_sessions must be greater than zero")); diff --git a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs new file mode 100644 index 00000000..cbcf588b --- /dev/null +++ b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs @@ -0,0 +1,211 @@ +//! One peer whose datagrams cannot be sent must not stop the server serving +//! the others. The failure is injected in `send_packet` through +//! `failing_peers`; a loopback peer cannot be made unroutable on demand. + +use super::*; +use cultnet_rs::{ + CultNetRawPayloadEncoding, CultNetRudpSocketMode, CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, +}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; + +#[derive(Clone)] +struct Clock(Arc); + +impl CultMeshRudpServerClock for Clock { + fn now_unix_millis(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } + fn now_monotonic_millis(&self) -> u64 { + self.0.load(Ordering::SeqCst) + } +} + +type Received = Arc>>; +type Server = CultMeshRudpDocumentServer< + Box Result<()>>, + Box Result>>, + Clock, +>; +type Client = CultNetRudpSocketTransportConnection; + +fn document(key: &str) -> CultNetRawDocumentRecord { + CultNetRawDocumentRecord { + schema_id: "test.raw.v1".into(), + record_key: key.into(), + stored_at: "2026-09-03T00:00:00Z".into(), + payload_encoding: CultNetRawPayloadEncoding::Messagepack, + payload: vec![0x81, 0xa1, 0x78, 0x2a], + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + } +} + +fn client(target: SocketAddr, id: u32) -> (Client, SocketAddr) { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.set_nonblocking(true).unwrap(); + let addr = socket.local_addr().unwrap(); + let client = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { + media_delivery: None, + runtime_id: "test-client".into(), + socket, + mode: CultNetRudpSocketMode::Client, + remote_addr: Some(target), + connection_id: id, + initial_sequence: 1, + resend_delay_ms: 10, + transport_id: None, + max_payload_bytes: None, + max_fragment_bytes: Some(1200), + max_pending_reliable_packets: Some(64), + media_reliable_expire_after_ms: None, + reconnect_policy: None, + }) + .unwrap(); + (client, addr) +} + +fn send(client: &mut Client, message: &CultNetMessage) { + client + .send( + "schema", + encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap(), + ) + .unwrap(); +} + +/// A server with a 1 s idle timeout and two connected peers, the first +/// returned being the one whose address sorts first. +fn two_peers() -> ( + Server, + Clock, + Received, + (Client, SocketAddr), + (Client, SocketAddr), +) { + let clock = Clock(Arc::new(AtomicU64::new(1_000))); + let received: Received = Arc::default(); + let sink_received = received.clone(); + let sink: Box Result<()>> = + Box::new(move |receipt| { + sink_received.lock().unwrap().push(receipt.message_id); + Ok(()) + }); + let source: Box< + dyn FnMut(&CultMeshRudpSnapshotQuery) -> Result>, + > = Box::new(|_| Ok(vec![document("catalog")])); + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0").unwrap(), + sink, + source, + clock.clone(), + CultMeshRudpDocumentServerOptions { + session_idle_timeout: Duration::from_secs(1), + resend_delay: Duration::from_millis(10), + ..Default::default() + }, + ) + .unwrap(); + let target = server.local_addr().unwrap(); + let (mut a, a_addr) = client(target, 7); + let (mut b, b_addr) = client(target, 7); + a.connect(Vec::new()).unwrap(); + b.connect(Vec::new()).unwrap(); + for _ in 0..500 { + server.poll_once().unwrap(); + a.receive_once().unwrap(); + b.receive_once().unwrap(); + if a.connected() && b.connected() { + break; + } + thread::sleep(Duration::from_millis(1)); + } + assert!(a.connected() && b.connected()); + for _ in 0..4 { + server.poll_once().unwrap(); + } + assert_eq!(server.session_count(), 2); + if a_addr < b_addr { + (server, clock, received, (a, a_addr), (b, b_addr)) + } else { + (server, clock, received, (b, b_addr), (a, a_addr)) + } +} + +#[test] +fn one_peer_whose_sends_fail_does_not_stop_the_server_serving_the_others() { + let (mut server, clock, received, (mut x, x_addr), (mut y, _)) = two_peers(); + server.failing_peers.insert(x_addr); + + // X's reply, ack and snapshot response all fail to send. X is visited first. + send( + &mut x, + &CultNetMessage::SnapshotRequest { + message_id: "x-snapshot".into(), + schema_ids: None, + record_keys: None, + }, + ); + send( + &mut y, + &CultNetMessage::DocumentPutRaw { + message_id: "y-put".into(), + document: document("y"), + }, + ); + send( + &mut y, + &CultNetMessage::SnapshotRequest { + message_id: "y-snapshot".into(), + schema_ids: None, + record_keys: None, + }, + ); + let mut y_response = None; + for step in 0..200 { + // The clock moves past X's resend delay so `maintain` has X's resend due. + clock.0.store(1_000 + step * 15, Ordering::SeqCst); + // Never `Err`: X's failed sends are its own lost datagrams. + server.poll_once().unwrap(); + if let Some(frame) = y.receive_once().unwrap() { + y_response = Some(frame); + break; + } + thread::sleep(Duration::from_millis(1)); + } + let frame = y_response.expect("Y receives its snapshot response"); + let response = + decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0) + .unwrap(); + assert!(matches!( + response, + CultNetMessage::SnapshotResponseRaw { ref message_id, .. } if message_id == "y-snapshot" + )); + assert!(server.send_failures() > 0); + assert_eq!(received.lock().unwrap().as_slice(), ["y-put"]); + assert!(server.maintain().is_ok()); + + // X's session ends by the idle rule that already ends sessions, not by the + // failure: Y stays active, X does not. + clock.0.store(1_900, Ordering::SeqCst); + send( + &mut y, + &CultNetMessage::DocumentPutRaw { + message_id: "y-put-2".into(), + document: document("y2"), + }, + ); + for _ in 0..20 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(server.session_count(), 2, "the failure ended no session"); + clock.0.store(2_500, Ordering::SeqCst); + let maintenance = server.maintain().unwrap(); + assert_eq!(maintenance.sessions_expired, 1); + assert_eq!(server.session_count(), 1); +} diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 3b348fe3..1d73216d 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -27,6 +27,9 @@ use crate::create_reconnect_policy; use crate::decode_cultnet_message_from_slice; use crate::encode_cultnet_message_to_vec; +#[cfg(test)] +mod send_failure_tests; + const RUDP_MAGIC: [u8; 4] = [0x43, 0x4e, 0x52, 0x30]; const RUDP_VERSION: u8 = 0; const RUDP_FIXED_HEADER_BYTES: usize = 36; @@ -1440,6 +1443,10 @@ pub struct CultNetRudpServerHub { pending_events: VecDeque, pub profile: CultNetTransportProfile, stats: CultNetTransportStats, + /// Peers whose every datagram fails to send, standing in for an unroutable + /// or full path that a loopback peer cannot be made to have. + #[cfg(test)] + failing_peers: BTreeSet, } impl CultNetRudpServerHub { @@ -1488,6 +1495,8 @@ impl CultNetRudpServerHub { pending_events: VecDeque::new(), profile, stats: CultNetTransportStats::default(), + #[cfg(test)] + failing_peers: BTreeSet::new(), }) } @@ -1796,12 +1805,26 @@ impl CultNetRudpServerHub { timed_out.into_iter().map(|(_, context)| context).collect() } + /// A failed send is that peer's lost datagram: counted, never an error. A + /// reliable packet stays pending and is resent; an unreliable one was + /// allowed to be lost. The peer's session ends by the rules that already + /// end sessions. Only an encode failure, the hub's own bug, is an error. fn send_packet(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { let wire = encode_rudp_packet(packet)?; - let sent = self.socket.send_to(&wire, remote_addr)?; - self.stats.bytes_sent += sent as u64; + match self.send_datagram(&wire, remote_addr) { + Ok(sent) => self.stats.bytes_sent += sent as u64, + Err(_) => self.stats.send_failures += 1, + } Ok(()) } + + fn send_datagram(&self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { + #[cfg(test)] + if self.failing_peers.contains(&remote_addr) { + return Err(std::io::Error::other("injected send failure")); + } + self.socket.send_to(wire, remote_addr) + } } impl CultNetRudpSocketTransportConnection { diff --git a/packages/cultnet-rs/src/rudp/send_failure_tests.rs b/packages/cultnet-rs/src/rudp/send_failure_tests.rs new file mode 100644 index 00000000..1a9b66a7 --- /dev/null +++ b/packages/cultnet-rs/src/rudp/send_failure_tests.rs @@ -0,0 +1,124 @@ +//! One peer whose datagrams cannot be sent must not stop the hub serving the +//! others. The failure is injected at `send_datagram` through `failing_peers`. + +use super::*; +use std::time::Duration; + +const CONNECTION_ID: u32 = 0x4355_4c54; + +fn socket() -> UdpSocket { + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + socket +} + +fn client(remote: SocketAddr) -> CultNetRudpSocketTransportConnection { + let mut options = + CultNetRudpSocketTransportOptions::client("peer", socket(), remote, CONNECTION_ID); + options.resend_delay_ms = 10; + CultNetRudpSocketTransportConnection::new(options).unwrap() +} + +fn connect( + hub: &mut CultNetRudpServerHub, + client: &mut CultNetRudpSocketTransportConnection, +) -> CultNetRudpServerSessionContext { + client.connect(Vec::new()).unwrap(); + let mut session = None; + for _ in 0..20 { + if let Some(CultNetRudpServerEvent::Connected { session: found }) = + hub.receive_event_once().unwrap() + { + session = Some(found); + break; + } + } + client.receive_once().unwrap(); + assert!(client.connected()); + session.expect("hub saw the Connect") +} + +/// Two connected peers, ordered so the first is the one the hub visits first +/// (the peer map is ordered by address). +fn two_peers() -> ( + CultNetRudpServerHub, + ( + CultNetRudpSocketTransportConnection, + CultNetRudpServerSessionContext, + ), + ( + CultNetRudpSocketTransportConnection, + CultNetRudpServerSessionContext, + ), +) { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let mut a = client(server_addr); + let mut b = client(server_addr); + let a_session = connect(&mut hub, &mut a); + let b_session = connect(&mut hub, &mut b); + if a_session.remote_addr < b_session.remote_addr { + (hub, (a, a_session), (b, b_session)) + } else { + (hub, (b, b_session), (a, a_session)) + } +} + +#[test] +fn a_failed_resend_to_one_peer_does_not_stop_the_others_resends() { + let (mut hub, (_x, x_session), (mut y, y_session)) = two_peers(); + // Both peers' first datagram is lost, so both have a reliable resend due. + hub.failing_peers.insert(x_session.remote_addr); + hub.failing_peers.insert(y_session.remote_addr); + hub.send(&x_session, "schema", b"for-x".to_vec()).unwrap(); + hub.send(&y_session, "schema", b"for-y".to_vec()).unwrap(); + assert_eq!(hub.stats().send_failures, 2); + + // Y's path recovers; X's stays dead. X is visited first. + hub.failing_peers.remove(&y_session.remote_addr); + std::thread::sleep(Duration::from_millis(40)); + hub.poll_resends().unwrap(); + + let mut delivered = None; + for _ in 0..20 { + if let Some(frame) = y.receive_once().unwrap() { + delivered = Some(frame); + break; + } + } + assert_eq!(delivered.expect("Y receives its resend").payload, b"for-y"); + assert!(hub.stats().send_failures >= 3); + assert_eq!(hub.sessions().len(), 2, "a send failure ends no session"); +} + +#[test] +fn a_failed_reply_to_one_peer_does_not_stop_the_hub_serving_the_others() { + let (mut hub, (mut x, x_session), (mut y, _y_session)) = two_peers(); + hub.failing_peers.insert(x_session.remote_addr); + + x.send("schema", b"from-x".to_vec()).unwrap(); + y.send("schema", b"from-y".to_vec()).unwrap(); + + let mut frames = Vec::new(); + for _ in 0..20 { + if let Some(CultNetRudpServerEvent::Frame { frame, .. }) = hub.receive_event_once().unwrap() + { + frames.push(frame.payload); + } + if frames.len() == 2 { + break; + } + } + frames.sort(); + assert_eq!(frames, vec![b"from-x".to_vec(), b"from-y".to_vec()]); + assert!( + hub.stats().send_failures >= 1, + "X's ack was counted as lost" + ); + assert_eq!(hub.sessions().len(), 2); +} diff --git a/packages/cultnet-rs/src/transport.rs b/packages/cultnet-rs/src/transport.rs index 71e714c4..f2c196d4 100644 --- a/packages/cultnet-rs/src/transport.rs +++ b/packages/cultnet-rs/src/transport.rs @@ -31,6 +31,11 @@ pub struct CultNetTransportStats { /// unadmitted sender, or a packet the session refused. A steady rate means /// a flow is being moved or scanned, not that this transport is failing. pub packets_dropped: u64, + /// Datagrams a multi-peer server could not send to one peer. Each is a lost + /// datagram attributed to that peer: reliable packets stay pending and are + /// resent, and no other peer's poll is affected. Zero for single-peer + /// transports, whose callers get the send error itself. + pub send_failures: u64, } #[derive(Clone, Debug, PartialEq, Eq)] From 8e609284d26c87f2ced656568497d75190eb0abd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:51:29 +0200 Subject: [PATCH 02/33] Ack Cut D (C#): a failed send to one peer is that peer's lost datagram CultNetRudpSocketTransportServer.SendPacket let a SocketException escape, so the first failed SendTo ended PollResends and TryReceiveOnce for every peer. SendPacket now counts the failure in CultNetTransportStats.SendFailures and returns; only an encode failure throws. The refusal goodbye's own catch is redundant and goes. The tests inject the failure at SendPacket through an internal FailingSendPeers set, exposed to GameCult.Networking.Tests by InternalsVisibleTo. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 40 ++++-- .../GameCult.Networking.csproj | 4 + .../CultNetRudpSendFailureTests.cs | 133 ++++++++++++++++++ 3 files changed, 166 insertions(+), 11 deletions(-) create mode 100644 tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index e98fdc03..4554eacd 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -39,6 +39,12 @@ namespace GameCult.Networking /// the peer, or a packet the session refuses. /// public long PacketsDropped { get; internal set; } + /// + /// Gets the number of datagrams a multi-peer server could not send to one peer. Each is + /// that peer's lost datagram: a reliable packet stays pending and is resent, and no other + /// peer is affected. Zero for single-peer transports, whose callers get the send error. + /// + public long SendFailures { get; internal set; } internal CultNetTransportStats Snapshot() { @@ -48,7 +54,8 @@ namespace GameCult.Networking FramesReceived = FramesReceived, BytesSent = BytesSent, BytesReceived = BytesReceived, - PacketsDropped = PacketsDropped + PacketsDropped = PacketsDropped, + SendFailures = SendFailures }; } } @@ -2734,14 +2741,8 @@ namespace GameCult.Networking if (outcome == null) { - try - { - SendPacket(existingPeer.RemoteEndPoint, existingPeer.Session.EndRefused()); - } - catch (SocketException) - { - // Best effort: the session ends whether or not the peer hears it. - } + // The session ends whether or not the peer hears the goodbye. + SendPacket(existingPeer.RemoteEndPoint, existingPeer.Session.EndRefused()); } else { @@ -2826,13 +2827,30 @@ namespace GameCult.Networking _socket.Dispose(); } + // A failed send is that peer's lost datagram: counted, never thrown. A reliable packet + // stays pending and is resent; an unreliable one was allowed to be lost. The peer's + // session ends by the rules that already end sessions. Only an encode failure, the + // listener's own bug, throws. private void SendPacket(EndPoint remoteEndPoint, CultNetRudpPacket packet) { var wire = CultNetRudpPacketCodec.Encode(packet); - var sent = _socket.SendTo(wire, remoteEndPoint); - _stats.BytesSent += sent; + try + { + if (FailingSendPeers.Contains(remoteEndPoint)) + throw new SocketException((int)SocketError.HostUnreachable); + var sent = _socket.SendTo(wire, remoteEndPoint); + _stats.BytesSent += sent; + } + catch (SocketException) + { + _stats.SendFailures++; + } } + // Test seam: peers whose every datagram fails to send, standing in for an unroutable or + // full path that a loopback peer cannot be made to have. + internal HashSet FailingSendPeers { get; } = new HashSet(); + private static string RemoteKey(EndPoint endpoint) { return endpoint.ToString() ?? string.Empty; diff --git a/src/GameCult.Networking/GameCult.Networking.csproj b/src/GameCult.Networking/GameCult.Networking.csproj index 1eef5653..08ef2ed6 100644 --- a/src/GameCult.Networking/GameCult.Networking.csproj +++ b/src/GameCult.Networking/GameCult.Networking.csproj @@ -15,6 +15,10 @@ + + + + diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs new file mode 100644 index 00000000..162ad4c6 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -0,0 +1,133 @@ +#nullable enable +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// One peer whose datagrams cannot be sent must not stop the listener serving the others. The + /// failure is injected at the listener's one send function through FailingSendPeers. + /// + public class CultNetRudpSendFailureTests + { + private const uint ConnectionId = 0x10203099; + + private sealed class Peer + { + public Socket Socket = null!; + public CultNetRudpSession Session = null!; + public CultNetRudpSocketServerPeer Server = null!; + public EndPoint EndPoint => Socket.LocalEndPoint!; + } + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return socket; + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + private static Peer Connect(CultNetRudpSocketTransportServer server, EndPoint serverEndPoint) + { + var peer = new Peer + { + Socket = Bind(), + Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }) + }; + peer.Socket.SendTo(CultNetRudpPacketCodec.Encode(peer.Session.CreateConnect(0)), serverEndPoint); + server.ReceiveOnce(); + peer.Session.Receive(Drain(peer.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + peer.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(peer.EndPoint)); + return peer; + } + + private static (CultNetRudpSocketTransportServer Server, EndPoint ServerEndPoint, Peer X, Peer Y) TwoPeers() + { + var serverSocket = Bind(); + var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-send-failure", + Socket = serverSocket, + ConnectionId = ConnectionId, + ResendDelayMs = 5 + }); + var x = Connect(server, serverSocket.LocalEndPoint!); + var y = Connect(server, serverSocket.LocalEndPoint!); + return (server, serverSocket.LocalEndPoint!, x, y); + } + + private static CultNetRudpPacket ReliableFrame(Peer peer, string text) => + peer.Session.Send("schema", Encoding.UTF8.GetBytes(text), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); + + [Test] + public void AFailedResendToOnePeerDoesNotStopTheOthersResends() + { + var (server, _, x, y) = TwoPeers(); + using (server) + { + // Both peers' first datagram is lost, so both have a reliable resend due. + server.FailingSendPeers.Add(x.EndPoint); + server.FailingSendPeers.Add(y.EndPoint); + server.SendSchema(x.Server, "for-x"); + server.SendSchema(y.Server, "for-y"); + Assert.That(server.Stats.SendFailures, Is.EqualTo(2)); + + // Y's path recovers; X's stays dead. X is visited first. + server.FailingSendPeers.Remove(y.EndPoint); + Thread.Sleep(30); + Assert.DoesNotThrow(() => server.PollResends()); + + var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList(); + Assert.That(resent, Has.Count.EqualTo(1)); + Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y")); + Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(3)); + Assert.That(server.Peers, Has.Count.EqualTo(2), "a send failure ends no session"); + } + } + + [Test] + public void AFailedAckToOnePeerDoesNotStopTheListenerServingTheOthers() + { + var (server, listenerEndPoint, x, y) = TwoPeers(); + using (server) + { + server.FailingSendPeers.Add(x.EndPoint); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint); + y.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(y, "from-y")), listenerEndPoint); + + var frames = new List(); + for (var attempt = 0; attempt < 20 && frames.Count < 2; attempt++) + { + Assert.DoesNotThrow(() => + { + if (server.ReceiveOnce() is { } frame) + frames.Add(Encoding.UTF8.GetString(frame.Frame.Payload)); + }); + } + + Assert.That(frames, Is.EquivalentTo(new[] { "from-x", "from-y" })); + Assert.That(server.Stats.SendFailures, Is.GreaterThanOrEqualTo(1), "X's ack was counted as lost"); + Assert.That(Drain(y.Socket).Select(p => p.PacketType), Does.Contain(CultNetRudpPacketType.Ack)); + Assert.That(server.Peers, Has.Count.EqualTo(2)); + } + } + } +} From 0b2f580c82a129aea80257564289e81f4fa0506c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:51:31 +0200 Subject: [PATCH 03/33] Ack Cut D (Python): a failed send to one peer no longer kills the RUDP thread CultMeshLocalServer._send_rudp_packet let an OSError from sendto escape _rudp_loop, which ended the RUDP thread for every peer. It now counts the failure (rudp_send_failures) and returns. The test patches socket.sendto to fail for one peer and serves another. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-py/src/cultmesh_py/server.py | 18 ++++- packages/cultmesh-py/tests/test_cultmesh.py | 69 +++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index e0c7fdd0..7ac2b82e 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -73,6 +73,14 @@ class CultMeshLocalServer: _stop: threading.Event = field(default_factory=threading.Event, init=False, repr=False) _thread: threading.Thread | None = field(default=None, init=False, repr=False) _rudp_thread: threading.Thread | None = field(default=None, init=False, repr=False) + _rudp_send_failures: int = field(default=0, init=False, repr=False) + + @property + def rudp_send_failures(self) -> int: + """Datagrams that could not be sent to a peer. Each is that peer's lost + datagram: a reliable packet stays pending and is resent, and no other + peer is affected.""" + return self._rudp_send_failures def __post_init__(self) -> None: if self.max_snapshot_documents is not None and self.max_snapshot_documents < 0: @@ -298,13 +306,19 @@ class CultMeshLocalServer: ): self._send_rudp_packet(rudp_socket, remote_addr, packet) - @staticmethod def _send_rudp_packet( + self, rudp_socket: socket.socket, remote_addr: tuple[str, int], packet: CultNetRudpPacket, ) -> None: - rudp_socket.sendto(encode_rudp_packet(packet), remote_addr) + # A failed send is that peer's lost datagram: counted, never raised. + # Raising here ended the RUDP thread for every peer. + wire = encode_rudp_packet(packet) + try: + rudp_socket.sendto(wire, remote_addr) + except OSError: + self._rudp_send_failures += 1 def _handle_connection_message( self, diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 9a697370..819dfc01 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -4498,5 +4498,74 @@ class CultMeshTests(unittest.TestCase): self.assertEqual(streams.latest_frame("mimir:kiyo-pro").sequence, 42) +class CultMeshRudpSendFailureTests(unittest.TestCase): + """One peer whose datagrams cannot be sent must not stop the RUDP thread + serving the others.""" + + def _client(self, server: Any, name: str) -> tuple[CultNetRudpSocketTransportConnection, tuple[str, int]]: + sock = bind_udp_socket() + client = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id=name, + socket=sock, + mode=CultNetRudpSocketMode.CLIENT, + connection_id=server.rudp_connection_id, + remote_addr=("127.0.0.1", server.port), + resend_delay_ms=25, + ) + ) + client.connect(b"join") + for _ in range(200): + client.receive_once() + if client.connected: + break + self.assertTrue(client.connected) + return client, sock.getsockname()[:2] + + def test_a_failed_send_to_one_peer_does_not_kill_the_rudp_thread(self) -> None: + import msgpack # type: ignore + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr): + raise OSError("injected send failure") + return original_sendto(sock, data, *args) + + request = msgpack.packb(hello(runtime_id="probe"), use_bin_type=True) + with patch.object(socket.socket, "sendto", sendto): + # The server's reply and ack to X both fail to send. + x.send("schema", request) + self._wait_for(lambda: server.rudp_send_failures > 0) + self.assertTrue(server._rudp_thread.is_alive()) + + y.send("schema", request) + response = None + for _ in range(200): + response = y.receive_once() + if response is not None: + break + time.sleep(0.005) + + self.assertIsNotNone(response, "Y is still served after X's send failed") + self.assertEqual(msgpack.unpackb(response.payload, raw=False)["runtimeId"], "mesh-server") + self.assertTrue(server._rudp_thread.is_alive()) + self.assertGreater(server.rudp_send_failures, 0) + finally: + server.stop() + + def _wait_for(self, predicate: Callable[[], bool]) -> None: + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.005) + self.fail("condition not reached") + + if __name__ == "__main__": unittest.main() From ce552b0e1b85551d8d08937391dbfad1f8e5272c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:56:51 +0200 Subject: [PATCH 04/33] Ack Cut D: fix the C# using and the Python request encoding in the new tests Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-py/tests/test_cultmesh.py | 2 +- tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 819dfc01..cccb2ecb 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -4536,7 +4536,7 @@ class CultMeshRudpSendFailureTests(unittest.TestCase): raise OSError("injected send failure") return original_sendto(sock, data, *args) - request = msgpack.packb(hello(runtime_id="probe"), use_bin_type=True) + request = hello(runtime_id="probe").to_bytes() with patch.object(socket.socket, "sendto", sendto): # The server's reply and ack to X both fail to send. x.send("schema", request) diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index 162ad4c6..359b842f 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -1,4 +1,5 @@ #nullable enable +using System; using System.Collections.Generic; using System.Linq; using System.Net; From 2125635dff905044950f10799f1034f4f6e2956d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 11:26:51 +0200 Subject: [PATCH 05/33] Ack Cut D: the document-server send-failure tests take initial_sequence as an Option Ack Cut 1c made CultNetRudpSocketTransportOptions.initial_sequence optional; the new client helper still passed a bare u32. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server/send_failure_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs index cbcf588b..1130a1ed 100644 --- a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs +++ b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs @@ -56,7 +56,7 @@ fn client(target: SocketAddr, id: u32) -> (Client, SocketAddr) { mode: CultNetRudpSocketMode::Client, remote_addr: Some(target), connection_id: id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 10, transport_id: None, max_payload_bytes: None, From 64ed64a8947c0888e2440862fe7963316152a3a3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:19:46 +0200 Subject: [PATCH 06/33] Ack Cut D D2 (Rust): a send that can never succeed is the caller's error inside a send and ends only that peer's session inside a poll The hub and the document server swallowed every send failure as a counted loss, so a 70,000-byte reliable frame returned Ok and was resent forever. Failures now split in two. Permanent means the datagram can never be sent as built (EMSGSIZE, EINVAL, EAFNOSUPPORT and their Winsock equivalents, is_permanent_send_error in cultnet-rs); everything else stays a counted loss. A caller-directed hub send returns the error and queues nothing: the packets it created are withdrawn and their sequences reissued, so the peer never sees a gap (a permanent failure after an earlier fragment left ends the session). Inside a poll the failure ends that peer's session with a goodbye whose reason names the error, reported to the hub caller as Disconnected, and never fails the poll. The document server ends the session the same way and reports a snapshot response it can never send as an application rejection. Tests pin each path, the classification per OS, BytesSent after a healthy send and a zero failure count for healthy sends. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 124 +++++++-- .../send_failure_tests.rs | 135 ++++++++++ packages/cultnet-rs/src/rudp.rs | 204 +++++++++++++-- .../cultnet-rs/src/rudp/send_failure_tests.rs | 247 ++++++++++++++++++ 4 files changed, 654 insertions(+), 56 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 98523e2b..518a44eb 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -3,7 +3,7 @@ use cultnet_rs::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRudpPacket, CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, CultNetWireContract, decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, - encode_rudp_packet, + encode_rudp_packet, is_permanent_send_error, }; use std::collections::BTreeMap; #[cfg(test)] @@ -215,6 +215,9 @@ pub struct CultMeshRudpDocumentServer { /// or full path that a loopback peer cannot be made to have. #[cfg(test)] failing_peers: BTreeSet, + /// Peers whose next datagram after this many fails permanently, once. + #[cfg(test)] + unsendable_after: BTreeMap, } impl CultMeshRudpDocumentServer @@ -246,6 +249,8 @@ where send_failures: 0, #[cfg(test)] failing_peers: BTreeSet::new(), + #[cfg(test)] + unsendable_after: BTreeMap::new(), }) } @@ -257,11 +262,14 @@ where self.packets_dropped } - /// Datagrams that could not be sent to a peer. Each is that peer's lost - /// datagram: a reliable packet stays pending and is resent, and the peer's - /// session ends by idle timeout, lifetime, refusal or Disconnect, never by - /// the failure. Never an error: one unreachable peer must not stop the - /// poll that serves the others. + /// Datagrams that could not be sent to a peer and may yet be: no route, full + /// buffers, a refused path. Each is that peer's lost datagram: a reliable + /// packet stays pending and is resent, and the peer's session ends by idle + /// timeout, lifetime, refusal or Disconnect, never by the failure. Never an + /// error: one unreachable peer must not stop the poll that serves the + /// others. A datagram that can never be sent as built + /// (`is_permanent_send_error`) is not counted here: it ends that peer's + /// session, and the peer's goodbye names the error. pub fn send_failures(&self) -> u64 { self.send_failures } @@ -357,9 +365,10 @@ where } }; - if let Some(reply) = result.reply { - self.send_packet(key.remote_addr, &reply)?; - } + let mut unsendable = match result.reply { + Some(reply) => self.send_packet(key.remote_addr, &reply)?, + None => None, + }; if result.disconnected { self.sessions.remove(&key); @@ -389,14 +398,17 @@ where } } - if packet.reliable { + if packet.reliable && unsendable.is_none() { let ack = self .sessions .get_mut(&key) .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before ACK"))? .session .create_ack(); - self.send_packet(key.remote_addr, &ack)?; + unsendable = self.send_packet(key.remote_addr, &ack)?; + } + if let Some(error) = unsendable { + self.end_unsendable_session(key, &error)?; } Ok(CultMeshRudpPollOutcome::Handled) } @@ -416,11 +428,17 @@ where let mut resends = Vec::new(); for (key, entry) in &mut self.sessions { for packet in entry.session.due_resends(now) { - resends.push((key.remote_addr, packet)); + resends.push((*key, packet)); } } - for (remote_addr, packet) in &resends { - self.send_packet(*remote_addr, packet)?; + for (key, packet) in &resends { + // A session ended earlier in this loop has nothing left to resend. + if !self.sessions.contains_key(key) { + continue; + } + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + self.end_unsendable_session(*key, &error)?; + } } Ok(CultMeshRudpMaintenance { sessions_expired: expired, @@ -482,7 +500,10 @@ where accept } }; - self.send_packet(key.remote_addr, &reply)?; + if let Some(error) = self.send_packet(key.remote_addr, &reply)? { + self.end_unsendable_session(key, &error)?; + return Ok(false); + } Ok(true) } @@ -618,7 +639,14 @@ where entry.admitted_payload_bytes = entry.admitted_payload_bytes.saturating_add(payload_bytes); for packet in &packets { - self.send_packet(key.remote_addr, packet)?; + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + return Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason: format!("snapshot response could not be sent: {error}"), + })); + } } } _ => {} @@ -633,7 +661,23 @@ where return Ok(()); }; let goodbye = entry.session.end_refused_session(); - self.send_packet(key.remote_addr, &goodbye) + self.send_packet(key.remote_addr, &goodbye)?; + Ok(()) + } + + /// The session owes its peer a packet that can never be sent as built. The + /// session ends, never the poll, and the goodbye names the error. + fn end_unsendable_session( + &mut self, + key: CultMeshRudpSessionKey, + error: &std::io::Error, + ) -> Result<()> { + let Some(mut entry) = self.sessions.remove(&key) else { + return Ok(()); + }; + let goodbye = entry.session.end_unsendable_session(error); + self.send_packet(key.remote_addr, &goodbye)?; + Ok(()) } fn payload_budget_allows(&self, key: CultMeshRudpSessionKey, bytes: usize) -> bool { @@ -659,20 +703,42 @@ where .is_some_and(|total| total <= self.options.max_admitted_payload_bytes) } - fn send_packet(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + /// A transient send failure is that peer's lost datagram: counted, never an + /// error. A permanent one (`is_permanent_send_error`) is returned as `Some` + /// for the caller to end the session over. Only an encode failure, the + /// server's own bug, is an error. + fn send_packet( + &mut self, + remote_addr: SocketAddr, + packet: &CultNetRudpPacket, + ) -> Result> { let wire = encode_rudp_packet(packet)?; - #[cfg(test)] - let sent = if self.failing_peers.contains(&remote_addr) { - Err(std::io::Error::other("injected send failure")) - } else { - self.socket.send_to(&wire, remote_addr) - }; - #[cfg(not(test))] - let sent = self.socket.send_to(&wire, remote_addr); - if sent.is_err() { - self.send_failures += 1; + match self.send_datagram(&wire, remote_addr) { + Ok(_) => {} + Err(error) if is_permanent_send_error(&error) => return Ok(Some(error)), + Err(_) => self.send_failures += 1, } - Ok(()) + Ok(None) + } + + fn send_datagram(&mut self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { + #[cfg(test)] + { + if self.failing_peers.contains(&remote_addr) { + return Err(std::io::Error::other("injected send failure")); + } + match self.unsendable_after.get(&remote_addr).copied() { + Some(0) => { + self.unsendable_after.remove(&remote_addr); + return Err(std::io::Error::from(ErrorKind::InvalidInput)); + } + Some(remaining) => { + self.unsendable_after.insert(remote_addr, remaining - 1); + } + None => {} + } + } + self.socket.send_to(wire, remote_addr) } } diff --git a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs index 1130a1ed..3554ab3c 100644 --- a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs +++ b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs @@ -209,3 +209,138 @@ fn one_peer_whose_sends_fail_does_not_stop_the_server_serving_the_others() { assert_eq!(maintenance.sessions_expired, 1); assert_eq!(server.session_count(), 1); } + +// A permanent send failure is one that can never succeed for the datagram as +// built (`is_permanent_send_error`). It ends that peer's session, never the +// poll, and the peer's goodbye names the error. + +fn put(id: &str) -> CultNetMessage { + CultNetMessage::DocumentPutRaw { + message_id: id.into(), + document: document(id), + } +} + +fn snapshot(id: &str) -> CultNetMessage { + CultNetMessage::SnapshotRequest { + message_id: id.into(), + schema_ids: None, + record_keys: None, + } +} + +fn goodbye_reason(client: &mut Client) -> Vec { + for _ in 0..50 { + client.receive_once().unwrap(); + if let Some(reason) = client.disconnect_reason() { + return reason.to_vec(); + } + thread::sleep(Duration::from_millis(2)); + } + panic!("the client was never told the session ended"); +} + +fn is_unsendable_reason(reason: &[u8]) -> bool { + reason.starts_with(b"packet could not be sent: ") + && reason.len() > b"packet could not be sent: ".len() +} + +#[test] +fn a_permanent_failure_acknowledging_a_document_ends_the_session_after_the_document_is_kept() { + let (mut server, _clock, received, (mut x, x_addr), (mut y, _)) = two_peers(); + server.unsendable_after.insert(x_addr, 0); + send(&mut x, &put("x-put")); + send(&mut y, &put("y-put")); + + for _ in 0..10 { + // Never `Err`: the failure ends X's session, not the poll. + server.poll_once().unwrap(); + } + let mut kept = received.lock().unwrap().clone(); + kept.sort(); + assert_eq!(kept, ["x-put", "y-put"]); + assert_eq!(server.session_count(), 1); + assert_eq!(server.send_failures(), 0); + assert!(is_unsendable_reason(&goodbye_reason(&mut x))); +} + +#[test] +fn a_permanent_failure_sending_a_snapshot_response_rejects_it_naming_the_error() { + let (mut server, _clock, _received, (mut x, x_addr), (_y, _)) = two_peers(); + server.unsendable_after.insert(x_addr, 0); + send(&mut x, &snapshot("x-snapshot")); + + let mut rejection = None; + for _ in 0..10 { + if let CultMeshRudpPollOutcome::ApplicationRejected(found) = server.poll_once().unwrap() { + rejection = Some(found); + break; + } + } + let rejection = rejection.expect("the response can never be sent"); + assert_eq!(rejection.operation, CultMeshRudpApplicationOperation::SnapshotRequest); + assert_eq!(rejection.message_id, "x-snapshot"); + assert!( + rejection + .reason + .starts_with("snapshot response could not be sent: ") + && rejection.reason.len() > "snapshot response could not be sent: ".len(), + "{}", + rejection.reason + ); + assert_eq!(server.session_count(), 1); +} + +#[test] +fn a_permanent_failure_in_resends_ends_only_that_peers_session() { + let (mut server, clock, received, (mut x, x_addr), (mut y, _)) = two_peers(); + // X's snapshot response is lost, so a resend is due; then the datagram can + // never be sent as built. + server.failing_peers.insert(x_addr); + send(&mut x, &snapshot("x-snapshot")); + for _ in 0..5 { + server.poll_once().unwrap(); + } + server.failing_peers.clear(); + server.unsendable_after.insert(x_addr, 0); + clock.0.store(1_200, Ordering::SeqCst); + + let maintenance = server.maintain().unwrap(); + assert!(maintenance.packets_resent > 0); + assert_eq!(server.session_count(), 1); + assert!(is_unsendable_reason(&goodbye_reason(&mut x))); + + send(&mut y, &put("y-put")); + for _ in 0..5 { + server.poll_once().unwrap(); + } + assert_eq!(received.lock().unwrap().as_slice(), ["y-put"]); +} + +#[test] +fn a_connect_whose_accept_can_never_be_sent_starts_no_session() { + let (mut server, _clock, _received, _x, _y) = two_peers(); + let (mut z, z_addr) = client(server.local_addr().unwrap(), 9); + server.unsendable_after.insert(z_addr, 0); + z.connect(Vec::new()).unwrap(); + + for _ in 0..10 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(server.session_count(), 2); + assert_eq!(server.packets_dropped(), 1); +} + +#[test] +fn healthy_sends_are_not_failures() { + let (mut server, _clock, _received, (mut x, _), (_y, _)) = two_peers(); + send(&mut x, &put("x-put")); + send(&mut x, &snapshot("x-snapshot")); + for _ in 0..10 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(server.send_failures(), 0); + assert_eq!(server.session_count(), 2); +} diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 4fb7ca29..1873c1fb 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -454,6 +454,28 @@ impl CultNetRudpSession { self.create_disconnect(RUDP_REFUSED_PACKET_REASON.to_vec()) } + /// Ends a session that owes its peer a packet that can never be sent as + /// built (see `is_permanent_send_error`). Resending it would fail forever, + /// and dropping it would leave a reliable sequence the peer waits on for + /// good, so the session cannot be kept. Returns the goodbye for the peer; + /// its reason names the error. + pub fn end_unsendable_session(&mut self, error: &std::io::Error) -> CultNetRudpPacket { + self.reset_peer_state(); + self.create_disconnect(format!("{RUDP_UNSENDABLE_PACKET_REASON}: {error}").into_bytes()) + } + + /// Undoes a send that never reached the wire: every packet it created at + /// or after `first_sequence`, pending or queued, leaves no trace and its + /// sequences are reissued, so the peer never sees a gap. Only valid while + /// nothing else has been sent since. + fn withdraw_send(&mut self, first_sequence: u32) { + self.pending_reliable + .retain(|sequence, _| *sequence < first_sequence); + self.queued_reliable + .retain(|(packet, _)| packet.sequence < first_sequence); + self.next_sequence = first_sequence; + } + pub fn create_connect(&mut self, now_ms: u64, payload: Vec) -> Result { // A session that has had a peer starts a new generation: nothing it // learned from that peer describes the one this Connect reaches, and @@ -1564,6 +1586,9 @@ pub struct CultNetRudpServerHub { /// or full path that a loopback peer cannot be made to have. #[cfg(test)] failing_peers: BTreeSet, + /// Peers whose next datagram after this many fails permanently, once. + #[cfg(test)] + unsendable_after: BTreeMap, } impl CultNetRudpServerHub { @@ -1614,6 +1639,8 @@ impl CultNetRudpServerHub { stats: CultNetTransportStats::default(), #[cfg(test)] failing_peers: BTreeSet::new(), + #[cfg(test)] + unsendable_after: BTreeMap::new(), }) } @@ -1654,7 +1681,7 @@ impl CultNetRudpServerHub { payload: Vec, ) -> Result<()> { let remote_addr = session.remote_addr; - let packets = { + let (first_sequence, packets) = { let peer = self .peers .get_mut(&remote_addr) @@ -1665,7 +1692,8 @@ impl CultNetRudpServerHub { session.session_generation )); } - peer.session.send_many( + let first_sequence = peer.session.next_sequence; + let packets = peer.session.send_many( channel_id, payload, channel_send_options( @@ -1675,10 +1703,25 @@ impl CultNetRudpServerHub { channel_delivery(&self.profile, channel_id), ), self.max_fragment_bytes, - )? + )?; + (first_sequence, packets) }; - for packet in packets { - self.send_packet(remote_addr, &packet)?; + for (index, packet) in packets.iter().enumerate() { + let Some(error) = self.send_packet(remote_addr, packet)? else { + continue; + }; + // A caller-directed send that can never succeed is the caller's + // error and queues nothing. Only when earlier fragments are already + // on the wire has the peer seen part of it, and then the session + // cannot be kept. + if index == 0 { + if let Some(peer) = self.peers.get_mut(&remote_addr) { + peer.session.withdraw_send(first_sequence); + } + } else { + self.end_unsendable_peer(remote_addr, &error)?; + } + return Err(error.into()); } self.stats.frames_sent += 1; Ok(()) @@ -1709,7 +1752,9 @@ impl CultNetRudpServerHub { .remove(&session.remote_addr) .expect("validated RUDP peer exists"); let packet = peer.session.create_disconnect(reason); - self.send_packet(session.remote_addr, &packet)?; + if let Some(error) = self.send_packet(session.remote_addr, &packet)? { + return Err(error.into()); + } Ok(true) } @@ -1767,7 +1812,7 @@ impl CultNetRudpServerHub { self.end_refusing_peer(remote_addr)?; return Ok(true); }; - self.send_packet(remote_addr, &reply)?; + self.send_in_poll(remote_addr, &reply)?; return Ok(true); } if !self.peers.contains_key(&remote_addr) && self.peers.len() >= self.max_peers { @@ -1796,7 +1841,11 @@ impl CultNetRudpServerHub { self.stats.packets_dropped += 1; return Ok(true); }; - self.send_packet(remote_addr, &accept)?; + if self.send_packet(remote_addr, &accept)?.is_some() { + // The peer cannot be answered, so no session starts. + self.stats.packets_dropped += 1; + return Ok(true); + } let replaced = self.peers.insert( remote_addr, CultNetRudpServerPeer { @@ -1830,14 +1879,17 @@ impl CultNetRudpServerHub { } else { None }; - if let Some(reply) = result.reply { - self.send_packet(remote_addr, &reply)?; - } - for ready in result.ready_to_send { - self.send_packet(remote_addr, &ready)?; - } - if let Some(ack) = ack { - self.send_packet(remote_addr, &ack)?; + let mut unsendable = None; + for packet in result + .reply + .iter() + .chain(&result.ready_to_send) + .chain(ack.iter()) + { + unsendable = self.send_packet(remote_addr, packet)?; + if unsendable.is_some() { + break; + } } if result.pong { self.pending_events.push_back(CultNetRudpServerEvent::Pong { @@ -1863,6 +1915,8 @@ impl CultNetRudpServerHub { reason: result.disconnect_reason, }); self.peers.remove(&remote_addr); + } else if let Some(error) = unsendable { + self.end_unsendable_peer(remote_addr, &error)?; } Ok(true) } @@ -1882,7 +1936,37 @@ impl CultNetRudpServerHub { session: peer.context, reason: goodbye.payload.clone(), }); - self.send_packet(remote_addr, &goodbye) + self.send_packet(remote_addr, &goodbye)?; + Ok(()) + } + + /// An admitted peer owes a packet that can never be sent as built. Inside a + /// poll that ends the peer's session, never the poll: the peer is told, and + /// the caller sees `Disconnected` with a reason that names the error. + fn end_unsendable_peer( + &mut self, + remote_addr: SocketAddr, + error: &std::io::Error, + ) -> Result<()> { + let Some(mut peer) = self.peers.remove(&remote_addr) else { + return Ok(()); + }; + let goodbye = peer.session.end_unsendable_session(error); + self.pending_events + .push_back(CultNetRudpServerEvent::Disconnected { + session: peer.context, + reason: goodbye.payload.clone(), + }); + self.send_packet(remote_addr, &goodbye)?; + Ok(()) + } + + /// A send inside a poll: a permanent failure ends that peer's session. + fn send_in_poll(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + if let Some(error) = self.send_packet(remote_addr, packet)? { + self.end_unsendable_peer(remote_addr, &error)?; + } + Ok(()) } pub fn poll_resends(&mut self) -> Result<()> { @@ -1897,7 +1981,10 @@ impl CultNetRudpServerHub { ); } for (remote_addr, packet) in packets { - self.send_packet(remote_addr, &packet)?; + // A peer ended earlier in this loop has nothing left to resend. + if self.peers.contains_key(&remote_addr) { + self.send_in_poll(remote_addr, &packet)?; + } } Ok(()) } @@ -1922,23 +2009,42 @@ impl CultNetRudpServerHub { timed_out.into_iter().map(|(_, context)| context).collect() } - /// A failed send is that peer's lost datagram: counted, never an error. A - /// reliable packet stays pending and is resent; an unreliable one was - /// allowed to be lost. The peer's session ends by the rules that already - /// end sessions. Only an encode failure, the hub's own bug, is an error. - fn send_packet(&mut self, remote_addr: SocketAddr, packet: &CultNetRudpPacket) -> Result<()> { + /// A transient send failure is that peer's lost datagram: counted, never an + /// error. A reliable packet stays pending and is resent; an unreliable one + /// was allowed to be lost. A permanent failure (`is_permanent_send_error`) + /// is returned as `Some`: the datagram can never be sent as built, and the + /// caller decides between refusing the send and ending the peer's session. + /// Only an encode failure, the hub's own bug, is an error. + fn send_packet( + &mut self, + remote_addr: SocketAddr, + packet: &CultNetRudpPacket, + ) -> Result> { let wire = encode_rudp_packet(packet)?; match self.send_datagram(&wire, remote_addr) { Ok(sent) => self.stats.bytes_sent += sent as u64, + Err(error) if is_permanent_send_error(&error) => return Ok(Some(error)), Err(_) => self.stats.send_failures += 1, } - Ok(()) + Ok(None) } - fn send_datagram(&self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { + fn send_datagram(&mut self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { #[cfg(test)] - if self.failing_peers.contains(&remote_addr) { - return Err(std::io::Error::other("injected send failure")); + { + if self.failing_peers.contains(&remote_addr) { + return Err(std::io::Error::other("injected send failure")); + } + match self.unsendable_after.get(&remote_addr).copied() { + Some(0) => { + self.unsendable_after.remove(&remote_addr); + return Err(std::io::Error::from(ErrorKind::InvalidInput)); + } + Some(remaining) => { + self.unsendable_after.insert(remote_addr, remaining - 1); + } + None => {} + } } self.socket.send_to(wire, remote_addr) } @@ -2428,6 +2534,50 @@ where /// the local caller as the disconnect reason. const RUDP_REFUSED_PACKET_REASON: &[u8] = b"session refused a packet"; const SESSION_TIMED_OUT_REASON: &[u8] = b"session timed out"; +/// Prefix of the goodbye reason for a session that owed a packet which can +/// never be sent; the error's own description follows it. +const RUDP_UNSENDABLE_PACKET_REASON: &str = "packet could not be sent"; + +/// Whether a failed `send_to` can never succeed for the datagram as built, so +/// resending it is pointless: it is too large for a datagram (`EMSGSIZE`), its +/// address is malformed for the socket (`EINVAL`), or belongs to another +/// address family (`EAFNOSUPPORT`). Everything else (no route, full buffers, +/// firewall drops, a refusal reported for an earlier datagram, an interrupted +/// call) is a property of the path or the moment and may pass, so it is a lost +/// datagram, not a reason to end a session. +pub fn is_permanent_send_error(error: &std::io::Error) -> bool { + // `EMSGSIZE` and `EAFNOSUPPORT`, or their Winsock equivalents. + #[cfg(any(target_os = "linux", target_os = "android"))] + const PERMANENT_OS_ERRORS: [i32; 2] = [90, 97]; + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly" + ))] + const PERMANENT_OS_ERRORS: [i32; 2] = [40, 47]; + #[cfg(windows)] + const PERMANENT_OS_ERRORS: [i32; 2] = [10040, 10047]; + #[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly", + windows + )))] + const PERMANENT_OS_ERRORS: [i32; 0] = []; + // `InvalidInput` is `EINVAL` and `WSAEINVAL`. + error.kind() == ErrorKind::InvalidInput + || error + .raw_os_error() + .is_some_and(|code| PERMANENT_OS_ERRORS.contains(&code)) +} /// A session cannot admit a Connect when its sequence space starts exhausted or /// its reliable queue holds nothing, so neither is a usable configuration. diff --git a/packages/cultnet-rs/src/rudp/send_failure_tests.rs b/packages/cultnet-rs/src/rudp/send_failure_tests.rs index 1a9b66a7..1c2cb3f2 100644 --- a/packages/cultnet-rs/src/rudp/send_failure_tests.rs +++ b/packages/cultnet-rs/src/rudp/send_failure_tests.rs @@ -122,3 +122,250 @@ fn a_failed_reply_to_one_peer_does_not_stop_the_hub_serving_the_others() { ); assert_eq!(hub.sessions().len(), 2); } + +// A permanent send failure is one that can never succeed for the datagram as +// built. A caller-directed send returns it and queues nothing; inside a poll it +// ends that peer's session and never the poll. + +fn is_unsendable_reason(reason: &[u8]) -> bool { + reason.starts_with(b"packet could not be sent: ") + && reason.len() > b"packet could not be sent: ".len() +} + +fn next_event(hub: &mut CultNetRudpServerHub) -> CultNetRudpServerEvent { + for _ in 0..20 { + if let Some(event) = hub.receive_event_once().unwrap() { + return event; + } + } + panic!("the hub raised no event"); +} + +fn one_peer( + max_fragment_bytes: Option, +) -> ( + CultNetRudpServerHub, + CultNetRudpSocketTransportConnection, + CultNetRudpServerSessionContext, +) { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + options.max_fragment_bytes = max_fragment_bytes; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let mut peer = client(server_addr); + let session = connect(&mut hub, &mut peer); + (hub, peer, session) +} + +#[test] +fn an_oversized_send_returns_the_error_and_queues_nothing() { + let (mut hub, mut peer, session) = one_peer(None); + + let error = hub + .send(&session, "schema", vec![7; 70_000]) + .expect_err("70,000 bytes cannot be one datagram"); + let io_error = error + .downcast_ref::() + .expect("the send error is the socket's own"); + assert!(is_permanent_send_error(io_error), "{io_error}"); + assert_eq!(hub.stats().send_failures, 0); + assert_eq!(hub.stats().frames_sent, 0); + assert_eq!(hub.sessions().len(), 1, "the caller's error ends no session"); + let remote = session.remote_addr; + assert_eq!( + hub.peers[&remote].session.outstanding_reliable_packet_count(), + 0, + "nothing is pending or queued to resend" + ); + + // The refused send consumed no sequence: the next frame is delivered in + // order instead of waiting behind a gap that never fills. + hub.send(&session, "schema", b"next".to_vec()).unwrap(); + let mut delivered = None; + for _ in 0..20 { + if let Some(frame) = peer.receive_once().unwrap() { + delivered = Some(frame); + break; + } + } + assert_eq!(delivered.expect("the next frame arrives").payload, b"next"); +} + +#[test] +fn a_send_that_fails_permanently_after_a_fragment_left_ends_the_session() { + let (mut hub, _peer, session) = one_peer(Some(1000)); + hub.unsendable_after.insert(session.remote_addr, 1); + + let error = hub + .send(&session, "schema", vec![7; 3000]) + .expect_err("the second fragment can never be sent"); + assert!(is_permanent_send_error( + error.downcast_ref::().unwrap() + )); + assert!(hub.sessions().is_empty()); + let CultNetRudpServerEvent::Disconnected { reason, .. } = next_event(&mut hub) else { + panic!("the caller sees the session end"); + }; + assert!(is_unsendable_reason(&reason)); +} + +#[test] +fn a_permanent_failure_in_resends_ends_only_that_peers_session() { + let (mut hub, (_x, x_session), (mut y, y_session)) = two_peers(); + // Both peers' first datagram is lost, so both have a reliable resend due. + hub.failing_peers.insert(x_session.remote_addr); + hub.failing_peers.insert(y_session.remote_addr); + hub.send(&x_session, "schema", b"for-x".to_vec()).unwrap(); + hub.send(&y_session, "schema", b"for-y".to_vec()).unwrap(); + + // X's datagram can now never be sent; Y's path recovers. X is visited first. + hub.failing_peers.clear(); + hub.unsendable_after.insert(x_session.remote_addr, 0); + std::thread::sleep(Duration::from_millis(40)); + hub.poll_resends().unwrap(); + + assert_eq!(hub.sessions(), vec![y_session.clone()]); + let CultNetRudpServerEvent::Disconnected { session, reason } = next_event(&mut hub) else { + panic!("the caller sees the session end"); + }; + assert_eq!(session, x_session); + assert!(is_unsendable_reason(&reason)); + let mut delivered = None; + for _ in 0..20 { + if let Some(frame) = y.receive_once().unwrap() { + delivered = Some(frame); + break; + } + } + assert_eq!(delivered.expect("Y receives its resend").payload, b"for-y"); +} + +#[test] +fn a_permanent_failure_replying_to_a_frame_ends_the_session_after_the_frame_is_delivered() { + let (mut hub, (mut x, x_session), (_y, _y_session)) = two_peers(); + hub.unsendable_after.insert(x_session.remote_addr, 0); + x.send("schema", b"from-x".to_vec()).unwrap(); + + let CultNetRudpServerEvent::Frame { frame, .. } = next_event(&mut hub) else { + panic!("the frame is delivered before the session ends"); + }; + assert_eq!(frame.payload, b"from-x"); + let CultNetRudpServerEvent::Disconnected { session, reason } = next_event(&mut hub) else { + panic!("the session ends"); + }; + assert_eq!(session, x_session); + assert!(is_unsendable_reason(&reason)); + assert_eq!(hub.sessions().len(), 1); +} + +#[test] +fn a_permanent_failure_answering_a_repeated_connect_ends_the_session() { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let raw = socket(); + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + initial_sequence: Some(5), + resend_delay_ms: 10, + max_pending_reliable_packets: None, + }); + let wire = encode_rudp_packet(&session.create_connect(0, Vec::new()).unwrap()).unwrap(); + + raw.send_to(&wire, server_addr).unwrap(); + let CultNetRudpServerEvent::Connected { session: admitted } = next_event(&mut hub) else { + panic!("the first Connect is admitted"); + }; + hub.unsendable_after.insert(raw.local_addr().unwrap(), 0); + raw.send_to(&wire, server_addr).unwrap(); + + let CultNetRudpServerEvent::Disconnected { + session: ended, + reason, + } = next_event(&mut hub) + else { + panic!("the reply to the repeat can never be sent, so the session ends"); + }; + assert_eq!(ended, admitted); + assert!(is_unsendable_reason(&reason)); + assert!(hub.sessions().is_empty()); +} + +#[test] +fn a_connect_whose_accept_can_never_be_sent_starts_no_session() { + let server_socket = socket(); + let server_addr = server_socket.local_addr().unwrap(); + let mut options = CultNetRudpServerHubOptions::new("hub", server_socket, CONNECTION_ID); + options.resend_delay_ms = 10; + let mut hub = CultNetRudpServerHub::new(options).unwrap(); + let mut peer = client(server_addr); + hub.unsendable_after + .insert(peer.socket.local_addr().unwrap(), 0); + + peer.connect(Vec::new()).unwrap(); + for _ in 0..5 { + assert_eq!(hub.receive_event_once().unwrap(), None); + } + assert!(hub.sessions().is_empty()); + assert_eq!(hub.stats().packets_dropped, 1); +} + +#[test] +fn a_healthy_send_counts_its_bytes_and_no_failure() { + let (mut hub, mut peer, session) = one_peer(None); + let before = hub.stats().bytes_sent; + let received_before = peer.stats().bytes_received; + hub.send(&session, "schema", b"hello".to_vec()).unwrap(); + for _ in 0..20 { + if peer.receive_once().unwrap().is_some() { + break; + } + } + let sent = hub.stats().bytes_sent - before; + assert!(sent > b"hello".len() as u64); + assert_eq!(sent, peer.stats().bytes_received - received_before); + assert_eq!(hub.stats().send_failures, 0); +} + +#[test] +fn only_a_datagram_that_can_never_be_sent_is_a_permanent_failure() { + use std::io::{Error, ErrorKind}; + assert!(is_permanent_send_error(&Error::from(ErrorKind::InvalidInput))); + assert!(!is_permanent_send_error(&Error::other("no os code"))); + assert!(!is_permanent_send_error(&Error::from(ErrorKind::WouldBlock))); + assert!(!is_permanent_send_error(&Error::from(ErrorKind::ConnectionReset))); + #[cfg(target_os = "linux")] + let (permanent, transient) = ( + // EMSGSIZE, EAFNOSUPPORT, EINVAL + [90, 97, 22], + // EPERM, EINTR, EAGAIN, EACCES, ENETDOWN, ENETUNREACH, EADDRNOTAVAIL, ENOBUFS, ECONNREFUSED, EHOSTUNREACH + [1, 4, 11, 13, 100, 101, 99, 105, 111, 113], + ); + #[cfg(windows)] + let (permanent, transient) = ( + // WSAEMSGSIZE, WSAEAFNOSUPPORT, WSAEINVAL + [10040, 10047, 10022], + // WSAEACCES, WSAEWOULDBLOCK, WSAENETDOWN, WSAENETUNREACH, WSAEADDRNOTAVAIL, WSAENOBUFS, WSAECONNRESET, + // WSAECONNREFUSED, WSAEHOSTUNREACH, WSAEINTR + [10013, 10035, 10050, 10051, 10049, 10055, 10054, 10061, 10065, 10004], + ); + #[cfg(any(target_os = "linux", windows))] + { + for code in permanent { + assert!( + is_permanent_send_error(&Error::from_raw_os_error(code)), + "os error {code} is permanent" + ); + } + for code in transient { + assert!( + !is_permanent_send_error(&Error::from_raw_os_error(code)), + "os error {code} is transient" + ); + } + } +} From 6cf92329b609342f6f70e5d36cd638035c5ef211 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:19:46 +0200 Subject: [PATCH 07/33] Ack Cut D D2 (C#): a send that can never succeed throws from a caller-directed send and ends only that peer's session inside a poll The listener swallowed every send failure as a counted loss. A send now splits into permanent (MessageSize, InvalidArgument, AddressFamilyNotSupported: the datagram can never be sent as built) and transient (everything else, still a counted loss). SendSchema throws the SocketException and queues nothing, with the packets it created withdrawn and their sequences reissued; a permanent failure after an earlier fragment left ends the session. Inside a poll the failure ends that peer's session, PeerDisconnected carries a reason naming the error, and the poll is never failed. Tests pin each path, the classification, BytesSent after a healthy send and a zero failure count. Co-Authored-By: Claude Sonnet 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 170 +++++++++++++-- .../CultNetRudpSendFailureTests.cs | 196 +++++++++++++++++- 2 files changed, 345 insertions(+), 21 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 83006d2f..f646a7a2 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -1459,6 +1459,53 @@ namespace GameCult.Networking return CreateDisconnect(RefusedPacketReason); } + /// + /// Whether a failed send can never succeed for the datagram as built, so resending it is + /// pointless: it is too large for a datagram (EMSGSIZE, WSAEMSGSIZE), its address is + /// malformed for the socket (EINVAL, WSAEINVAL), or belongs to another address family + /// (EAFNOSUPPORT, WSAEAFNOSUPPORT). Everything else (no route, full buffers, firewall drops, + /// a refusal reported for an earlier datagram, an interrupted call) is a property of the path + /// or the moment and may pass, so it is a lost datagram, not a reason to end a session. + /// + internal static bool IsPermanentSendError(SocketException error) => + error.SocketErrorCode == SocketError.MessageSize || + error.SocketErrorCode == SocketError.InvalidArgument || + error.SocketErrorCode == SocketError.AddressFamilyNotSupported; + + internal const string UnsendablePacketReason = "packet could not be sent"; + + /// + /// Ends a session that owes its peer a packet that can never be sent as built. Resending it + /// would fail forever, and dropping it would leave a reliable sequence the peer waits on for + /// good, so the session cannot be kept. The goodbye's reason names the error. + /// + internal CultNetRudpPacket EndUnsendable(SocketException error) + { + ResetPeerState(); + return CreateDisconnect(Encoding.UTF8.GetBytes(UnsendablePacketReason + ": " + error.Message)); + } + + internal uint NextSequence => _nextSequence; + + /// + /// Undoes a send that never reached the wire: every packet it created at or after + /// , pending or queued, leaves no trace and its sequences are + /// reissued, so the peer never sees a gap. Only valid while nothing else has been sent since. + /// + internal void WithdrawSend(uint firstSequence) + { + lock (_pendingReliableGate) + { + foreach (var sequence in _pendingReliable.Keys.Where(sequence => sequence >= firstSequence).ToArray()) + _pendingReliable.Remove(sequence); + var kept = _queuedReliable.Where(packet => packet.Sequence < firstSequence).ToArray(); + _queuedReliable.Clear(); + foreach (var packet in kept) + _queuedReliable.Enqueue(packet); + } + _nextSequence = firstSequence; + } + /// /// Creates a packet carrying a transport-level disconnect reason. /// @@ -2733,8 +2780,25 @@ namespace GameCult.Networking public void Send(CultNetRudpSocketServerPeer peer, string channelId, byte[] payload) { if (peer == null) throw new ArgumentNullException(nameof(peer)); + SocketException? unsendable = null; + var failedIndex = 0; lock (peer.SessionGate) - SendPackets(peer, peer.Session.SendMany(channelId, payload, ChannelSendOptions(channelId), _maxFragmentBytes)); + { + var firstSequence = peer.Session.NextSequence; + var packets = peer.Session.SendMany(channelId, payload, ChannelSendOptions(channelId), _maxFragmentBytes); + unsendable = SendPackets(peer, packets, out failedIndex); + // A caller-directed send that can never succeed is the caller's error and queues + // nothing. Only when earlier fragments are already on the wire has the peer seen + // part of it, and then the session cannot be kept. + if (unsendable != null && failedIndex == 0) + peer.Session.WithdrawSend(firstSequence); + } + if (unsendable != null) + { + if (failedIndex > 0) + EndUnsendablePeer(peer, unsendable); + throw unsendable; + } _stats.FramesSent++; } @@ -2844,10 +2908,13 @@ namespace GameCult.Networking // a new generation. if (admitted != null) { + var repeats = false; + SocketException? unsendableReply = null; lock (admitted.SessionGate) { if (admitted.Session.ConnectRepeats(packet)) { + repeats = true; CultNetRudpPacket reply; try { @@ -2858,10 +2925,15 @@ namespace GameCult.Networking _stats.PacketsDropped++; return true; } - SendPacket(admitted.RemoteEndPoint, reply); - return true; + unsendableReply = SendPacket(admitted.RemoteEndPoint, reply); } } + if (repeats) + { + if (unsendableReply != null) + EndUnsendablePeer(admitted, unsendableReply); + return true; + } } var peer = new CultNetRudpSocketServerPeer( @@ -2874,9 +2946,16 @@ namespace GameCult.Networking ResendDelayMs = _resendDelayMs, MaxPendingReliablePackets = _maxPendingReliablePackets })); - _peers[peerKey] = peer; + SocketException? unsendableAccept; lock (peer.SessionGate) - SendPacket(peer.RemoteEndPoint, peer.Session.AcceptConnect(packet, NowMs(), _acceptPayload)); + unsendableAccept = SendPacket(peer.RemoteEndPoint, peer.Session.AcceptConnect(packet, NowMs(), _acceptPayload)); + if (unsendableAccept != null) + { + // The peer cannot be answered, so no session starts. + _stats.PacketsDropped++; + return true; + } + _peers[peerKey] = peer; if (admitted != null) { admitted.DisconnectReason = ReplacedPeerReason; @@ -2893,6 +2972,7 @@ namespace GameCult.Networking CultNetRudpReceiveResult? outcome; CultNetRudpPacket? acknowledgement = null; + SocketException? unsendable = null; lock (existingPeer.SessionGate) { try @@ -2915,8 +2995,8 @@ namespace GameCult.Networking else { if (outcome.Reply != null) - SendPacket(existingPeer.RemoteEndPoint, outcome.Reply); - SendPackets(existingPeer, outcome.ReadyToSend); + unsendable = SendPacket(existingPeer.RemoteEndPoint, outcome.Reply); + unsendable ??= SendPackets(existingPeer, outcome.ReadyToSend, out _); if (packet.PacketType == CultNetRudpPacketType.Data) acknowledgement = existingPeer.Session.CreateAckForReceived(packet.Sequence); } @@ -2953,8 +3033,10 @@ namespace GameCult.Networking } delivered = _deliveredFrames.Count > 0 ? _deliveredFrames.Dequeue() : null; - if (acknowledgement != null) - SendPacket(existingPeer.RemoteEndPoint, acknowledgement); + if (acknowledgement != null && unsendable == null) + unsendable = SendPacket(existingPeer.RemoteEndPoint, acknowledgement); + if (unsendable != null) + EndUnsendablePeer(existingPeer, unsendable); return true; } @@ -2968,19 +3050,48 @@ namespace GameCult.Networking return; foreach (var peer in _peers.Values.ToArray()) { + SocketException? unsendable; lock (peer.SessionGate) - SendPackets(peer, peer.Session.DueResends(NowMs())); + unsendable = SendPackets(peer, peer.Session.DueResends(NowMs()), out _); + if (unsendable != null) + EndUnsendablePeer(peer, unsendable); } } - private void SendPackets(CultNetRudpSocketServerPeer peer, IReadOnlyList packets) + // Stops at the first packet that can never be sent as built and returns its error and + // index; a transient failure is a lost datagram and the loop goes on. + private SocketException? SendPackets(CultNetRudpSocketServerPeer peer, IReadOnlyList packets, out int failedIndex) { + failedIndex = 0; for (var index = 0; index < packets.Count; index++) { - SendPacket(peer.RemoteEndPoint, packets[index]); + var unsendable = SendPacket(peer.RemoteEndPoint, packets[index]); + if (unsendable != null) + { + failedIndex = index; + return unsendable; + } if ((index + 1) % WireBurstPackets == 0 && index + 1 < packets.Count) Thread.Yield(); } + return null; + } + + // An admitted peer owes a packet that can never be sent as built. Inside a poll that ends the + // peer's session, never the poll: the peer is told, and PeerDisconnected carries a reason that + // names the error. + private void EndUnsendablePeer(CultNetRudpSocketServerPeer peer, SocketException error) + { + var key = RemoteKey(peer.RemoteEndPoint); + if (!_peers.TryGetValue(key, out var current) || !ReferenceEquals(current, peer)) + return; + _peers.Remove(key); + CultNetRudpPacket goodbye; + lock (peer.SessionGate) + goodbye = peer.Session.EndUnsendable(error); + peer.DisconnectReason = goodbye.Payload; + SendPacket(peer.RemoteEndPoint, goodbye); + PeerDisconnected?.Invoke(peer); } /// @@ -2995,30 +3106,51 @@ namespace GameCult.Networking _socket.Dispose(); } - // A failed send is that peer's lost datagram: counted, never thrown. A reliable packet - // stays pending and is resent; an unreliable one was allowed to be lost. The peer's - // session ends by the rules that already end sessions. Only an encode failure, the - // listener's own bug, throws. - private void SendPacket(EndPoint remoteEndPoint, CultNetRudpPacket packet) + // A transient send failure is that peer's lost datagram: counted, never thrown. A reliable + // packet stays pending and is resent; an unreliable one was allowed to be lost. A permanent + // failure (CultNetRudpSession.IsPermanentSendError) is returned: the datagram can never be + // sent as built, and the caller decides between refusing the send and ending the peer's + // session. Only an encode failure, the listener's own bug, throws. + private SocketException? SendPacket(EndPoint remoteEndPoint, CultNetRudpPacket packet) { var wire = CultNetRudpPacketCodec.Encode(packet); try { if (FailingSendPeers.Contains(remoteEndPoint)) throw new SocketException((int)SocketError.HostUnreachable); + if (UnsendableAfter.TryGetValue(remoteEndPoint, out var remaining)) + { + if (remaining > 0) + { + UnsendableAfter[remoteEndPoint] = remaining - 1; + } + else + { + UnsendableAfter.Remove(remoteEndPoint); + throw new SocketException((int)SocketError.MessageSize); + } + } var sent = _socket.SendTo(wire, remoteEndPoint); _stats.BytesSent += sent; } + catch (SocketException error) when (CultNetRudpSession.IsPermanentSendError(error)) + { + return error; + } catch (SocketException) { _stats.SendFailures++; } + return null; } - // Test seam: peers whose every datagram fails to send, standing in for an unroutable or - // full path that a loopback peer cannot be made to have. + // Test seams, standing in for paths a loopback peer cannot be made to have. Peers whose every + // datagram fails to send transiently: internal HashSet FailingSendPeers { get; } = new HashSet(); + // Peers whose next datagram after this many fails permanently, once. + internal Dictionary UnsendableAfter { get; } = new Dictionary(); + private static string RemoteKey(EndPoint endpoint) { return endpoint.ToString() ?? string.Empty; diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index 359b842f..ba6cba1f 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -60,7 +60,7 @@ namespace GameCult.Networking.Tests return peer; } - private static (CultNetRudpSocketTransportServer Server, EndPoint ServerEndPoint, Peer X, Peer Y) TwoPeers() + private static (CultNetRudpSocketTransportServer Server, EndPoint ServerEndPoint, Peer X, Peer Y) TwoPeers(int? maxFragmentBytes = null) { var serverSocket = Bind(); var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions @@ -68,7 +68,8 @@ namespace GameCult.Networking.Tests RuntimeId = "csharp-send-failure", Socket = serverSocket, ConnectionId = ConnectionId, - ResendDelayMs = 5 + ResendDelayMs = 5, + MaxFragmentBytes = maxFragmentBytes }); var x = Connect(server, serverSocket.LocalEndPoint!); var y = Connect(server, serverSocket.LocalEndPoint!); @@ -130,5 +131,196 @@ namespace GameCult.Networking.Tests Assert.That(server.Peers, Has.Count.EqualTo(2)); } } + + // A permanent send failure is one that can never succeed for the datagram as built. A + // caller-directed send throws it and queues nothing; inside a poll it ends that peer's + // session and never the poll. + + private static bool IsUnsendableReason(byte[]? reason) + { + var text = Encoding.UTF8.GetString(reason ?? Array.Empty()); + return text.StartsWith("packet could not be sent: ", StringComparison.Ordinal) + && text.Length > "packet could not be sent: ".Length; + } + + private static List RecordDisconnects(CultNetRudpSocketTransportServer server) + { + var ended = new List(); + server.PeerDisconnected += ended.Add; + return ended; + } + + [Test] + public void AnOversizedSendThrowsTheErrorAndQueuesNothing() + { + var (server, _, x, y) = TwoPeers(); + using (server) + { + var error = Assert.Throws(() => server.SendSchema(x.Server, new byte[70_000]))!; + Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True, error.SocketErrorCode.ToString()); + Assert.That(server.Stats.SendFailures, Is.Zero); + Assert.That(server.Stats.FramesSent, Is.Zero); + Assert.That(server.Peers, Has.Count.EqualTo(2), "the caller's error ends no session"); + Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.Zero, "nothing is pending or queued to resend"); + + // The refused send consumed no sequence: the next frame is delivered in order instead + // of waiting behind a gap that never fills. + server.SendSchema(x.Server, "next"); + var data = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Data); + var delivered = x.Session.Receive(data, 1).Delivered; + Assert.That(delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)), Is.EqualTo(new[] { "next" })); + Assert.That(y.Server.Session.OutstandingReliablePacketCount, Is.Zero); + } + } + + [Test] + public void ASendThatFailsPermanentlyAfterAFragmentLeftEndsTheSession() + { + var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var ended = RecordDisconnects(server); + server.UnsendableAfter[x.EndPoint] = 1; + + var error = Assert.Throws(() => server.SendSchema(x.Server, new byte[3000]))!; + Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + } + } + + [Test] + public void APermanentFailureInResendsEndsOnlyThatPeersSession() + { + var (server, _, x, y) = TwoPeers(); + using (server) + { + var ended = RecordDisconnects(server); + server.FailingSendPeers.Add(x.EndPoint); + server.FailingSendPeers.Add(y.EndPoint); + server.SendSchema(x.Server, "for-x"); + server.SendSchema(y.Server, "for-y"); + + // X's datagram can now never be sent as built; Y's path recovers. X is visited first. + server.FailingSendPeers.Clear(); + server.UnsendableAfter[x.EndPoint] = 0; + Thread.Sleep(30); + Assert.DoesNotThrow(() => server.PollResends()); + + Assert.That(server.Peers, Is.EqualTo(new[] { y.Server })); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + var resent = Drain(y.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Data).ToList(); + Assert.That(resent, Has.Count.EqualTo(1)); + Assert.That(Encoding.UTF8.GetString(resent[0].Payload!), Is.EqualTo("for-y")); + var goodbye = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Disconnect); + Assert.That(IsUnsendableReason(goodbye.Payload), Is.True); + } + } + + [Test] + public void APermanentFailureAcknowledgingAFrameEndsTheSessionAfterTheFrameIsDelivered() + { + var (server, listenerEndPoint, x, _) = TwoPeers(); + using (server) + { + var ended = RecordDisconnects(server); + server.UnsendableAfter[x.EndPoint] = 0; + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint); + + CultNetRudpSocketServerFrame? frame = null; + Assert.DoesNotThrow(() => frame = server.ReceiveOnce()); + + Assert.That(frame, Is.Not.Null, "the frame is delivered before the session ends"); + Assert.That(Encoding.UTF8.GetString(frame!.Frame.Payload), Is.EqualTo("from-x")); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + Assert.That(server.Stats.SendFailures, Is.Zero); + } + } + + [Test] + public void APermanentFailureAnsweringARepeatedConnectEndsTheSession() + { + var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-send-failure", + Socket = serverSocket, + ConnectionId = ConnectionId, + ResendDelayMs = 5 + }); + var ended = RecordDisconnects(server); + var peer = Bind(); + var connect = CultNetRudpPacketCodec.Encode( + new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0)); + peer.SendTo(connect, serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + var admitted = server.Peers.Single(); + + server.UnsendableAfter[peer.LocalEndPoint!] = 0; + peer.SendTo(connect, serverSocket.LocalEndPoint!); + Assert.DoesNotThrow(() => server.ReceiveOnce()); + + Assert.That(server.Peers, Is.Empty); + Assert.That(ended, Is.EqualTo(new[] { admitted })); + Assert.That(IsUnsendableReason(admitted.DisconnectReason), Is.True); + } + + [Test] + public void AConnectWhoseAcceptCanNeverBeSentStartsNoSession() + { + var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-send-failure", + Socket = serverSocket, + ConnectionId = ConnectionId, + ResendDelayMs = 5 + }); + var peer = Bind(); + server.UnsendableAfter[peer.LocalEndPoint!] = 0; + peer.SendTo( + CultNetRudpPacketCodec.Encode(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }).CreateConnect(0)), + serverSocket.LocalEndPoint!); + + Assert.DoesNotThrow(() => server.ReceiveOnce()); + + Assert.That(server.Peers, Is.Empty); + Assert.That(server.Stats.PacketsDropped, Is.EqualTo(1)); + } + + [Test] + public void AHealthySendCountsItsBytesAndNoFailure() + { + var (server, _, x, _) = TwoPeers(); + using (server) + { + var before = server.Stats.BytesSent; + server.SendSchema(x.Server, "hello"); + var received = Drain(x.Socket).Sum(p => CultNetRudpPacketCodec.Encode(p).Length); + Assert.That(received, Is.GreaterThan("hello".Length)); + Assert.That(server.Stats.BytesSent - before, Is.EqualTo(received)); + Assert.That(server.Stats.SendFailures, Is.Zero); + } + } + + [Test] + public void OnlyADatagramThatCanNeverBeSentIsAPermanentFailure() + { + foreach (var permanent in new[] { SocketError.MessageSize, SocketError.InvalidArgument, SocketError.AddressFamilyNotSupported }) + Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)permanent)), Is.True, permanent.ToString()); + foreach (var transient in new[] + { + SocketError.HostUnreachable, SocketError.NetworkUnreachable, SocketError.NoBufferSpaceAvailable, + SocketError.WouldBlock, SocketError.ConnectionReset, SocketError.ConnectionRefused, + SocketError.AccessDenied, SocketError.NetworkDown, SocketError.AddressNotAvailable, + SocketError.Interrupted, SocketError.TimedOut, SocketError.Shutdown + }) + Assert.That(CultNetRudpSession.IsPermanentSendError(new SocketException((int)transient)), Is.False, transient.ToString()); + } } } From 1c71d98c71f616bd029a1417ff9b4f7916ba0a7b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:19:47 +0200 Subject: [PATCH 08/33] Ack Cut D D1 and D2 (Python): a vanished peer is idle, and a send that can never succeed ends only that peer's session D1: on Windows the closed port of a vanished peer is reported by the next recvfrom as ConnectionResetError (WinError 10054), and the RUDP thread returned on it. It is now idle, as in the Rust document server, the Rust hub and the C# listener. Both tests fail on Windows without the fix and pass with it. D2: is_permanent_send_error (EMSGSIZE, EINVAL, EAFNOSUPPORT and their Winsock equivalents) separates a datagram that can never be sent as built from a lost one. The server ends that peer's session with a goodbye whose reason names the error and never the thread; transient failures stay counted losses. Co-Authored-By: Claude Sonnet 5.5 --- .../cultmesh-py/src/cultmesh_py/server.py | 83 ++++++++++++----- packages/cultmesh-py/tests/test_cultmesh.py | 93 +++++++++++++++++++ .../cultnet-py/src/cultnet_py/__init__.py | 2 + .../cultnet-py/src/cultnet_py/transport.py | 35 +++++++ packages/cultnet-py/tests/test_cultnet.py | 22 +++++ 5 files changed, 213 insertions(+), 22 deletions(-) diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index ff6d998d..41d9d9e9 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -20,6 +20,7 @@ from cultnet_py import ( create_tcp_framed_transport_profile, decode_rudp_packet, encode_rudp_packet, + is_permanent_send_error, wire_message_schema_descriptors, ) @@ -79,7 +80,9 @@ class CultMeshLocalServer: def rudp_send_failures(self) -> int: """Datagrams that could not be sent to a peer. Each is that peer's lost datagram: a reliable packet stays pending and is resent, and no other - peer is affected.""" + peer is affected. A datagram that can never be sent as built + (`is_permanent_send_error`) is not counted here: it ends that peer's + session, and the peer's goodbye names the error.""" return self._rudp_send_failures def __post_init__(self) -> None: @@ -216,7 +219,10 @@ class CultMeshLocalServer: while not self._stop.is_set(): try: wire, remote_addr = rudp_socket.recvfrom(65535) - except TimeoutError: + except (TimeoutError, ConnectionResetError): + # Windows reports an earlier ICMP port-unreachable, from a peer that + # has gone away, as an error on the next receive. It names no datagram + # of this read, so it is idle, not the end of the thread. self._poll_rudp_resends(rudp_socket, peers) continue except OSError: @@ -245,20 +251,23 @@ class CultMeshLocalServer: ) ) peers[remote_addr] = peer - self._send_rudp_packet( + unsendable = self._send_rudp_packet( rudp_socket, remote_addr, peer.session.accept_connect(packet, now_ms, b"cultmesh-local-rudp"), ) + if unsendable is not None: + self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) continue if peer is None: continue result = peer.session.receive(packet, now_ms) - if result.reply is not None: - self._send_rudp_packet(rudp_socket, remote_addr, result.reply) - for ready in result.ready_to_send: - self._send_rudp_packet(rudp_socket, remote_addr, ready) + unsendable: OSError | None = None + for outgoing in (*([result.reply] if result.reply is not None else []), *result.ready_to_send): + unsendable = self._send_rudp_packet(rudp_socket, remote_addr, outgoing) + if unsendable is not None: + break if result.disconnected: peers.pop(remote_addr, None) continue @@ -273,18 +282,23 @@ class CultMeshLocalServer: continue responses = self._handle_connection_message(message, peer.subscriptions) for response in responses: - self._send_rudp_schema_frame( - rudp_socket, - remote_addr, - peer.session, - msgpack.packb(response, use_bin_type=True), - ) - if packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered: - self._send_rudp_packet( + if unsendable is None: + unsendable = self._send_rudp_schema_frame( + rudp_socket, + remote_addr, + peer.session, + msgpack.packb(response, use_bin_type=True), + ) + if unsendable is None and ( + packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered + ): + unsendable = self._send_rudp_packet( rudp_socket, remote_addr, peer.session.create_ack_for_received(packet.sequence), ) + if unsendable is not None: + self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) def _poll_rudp_resends( self, @@ -294,7 +308,10 @@ class CultMeshLocalServer: now_ms = _now_ms() for remote_addr, peer in list(peers.items()): for packet in peer.session.due_resends(now_ms): - self._send_rudp_packet(rudp_socket, remote_addr, packet) + unsendable = self._send_rudp_packet(rudp_socket, remote_addr, packet) + if unsendable is not None: + self._end_unsendable_peer(rudp_socket, peers, remote_addr, unsendable) + break def _send_rudp_schema_frame( self, @@ -302,27 +319,49 @@ class CultMeshLocalServer: remote_addr: tuple[str, int], session: CultNetRudpSession, payload: bytes, - ) -> None: + ) -> OSError | None: for packet in session.send_many( "schema", payload, CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=_now_ms()), ): - self._send_rudp_packet(rudp_socket, remote_addr, packet) + unsendable = self._send_rudp_packet(rudp_socket, remote_addr, packet) + if unsendable is not None: + return unsendable + return None def _send_rudp_packet( self, rudp_socket: socket.socket, remote_addr: tuple[str, int], packet: CultNetRudpPacket, - ) -> None: - # A failed send is that peer's lost datagram: counted, never raised. - # Raising here ended the RUDP thread for every peer. + ) -> OSError | None: + # A transient send failure is that peer's lost datagram: counted, never + # raised (raising here ended the RUDP thread for every peer). A permanent + # one (`is_permanent_send_error`) is returned: the datagram can never be + # sent as built, and the caller ends that peer's session. wire = encode_rudp_packet(packet) try: rudp_socket.sendto(wire, remote_addr) - except OSError: + except OSError as error: + if is_permanent_send_error(error): + return error self._rudp_send_failures += 1 + return None + + def _end_unsendable_peer( + self, + rudp_socket: socket.socket, + peers: dict[tuple[str, int], _RudpPeerConnection], + remote_addr: tuple[str, int], + error: OSError, + ) -> None: + """The session owes its peer a packet that can never be sent as built. It ends, + never the thread, and the peer's goodbye names the error.""" + peer = peers.pop(remote_addr, None) + if peer is None: + return + self._send_rudp_packet(rudp_socket, remote_addr, peer.session.end_unsendable_session(error)) def _handle_connection_message( self, diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index cccb2ecb..18243923 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import json import os import socket @@ -4558,6 +4559,98 @@ class CultMeshRudpSendFailureTests(unittest.TestCase): finally: server.stop() + def _served(self, y: CultNetRudpSocketTransportConnection) -> Any: + import msgpack # type: ignore + + y.send("schema", hello(runtime_id="probe").to_bytes()) + for _ in range(200): + response = y.receive_once() + if response is not None: + return msgpack.unpackb(response.payload, raw=False) + time.sleep(0.005) + return None + + def test_a_vanished_peer_does_not_kill_the_rudp_thread(self) -> None: + # On Windows the closed port of a peer that went away is reported as + # ConnectionResetError (WinError 10054) by the next receive. It can only fail there. + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + x, _ = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + x.send("schema", hello(runtime_id="probe").to_bytes()) + x.socket.close() + time.sleep(1.0) # the server's reply and resends to X draw ICMP port-unreachable + + self.assertTrue(server._rudp_thread.is_alive()) + served = self._served(y) + self.assertIsNotNone(served, "Y is still served after X vanished") + self.assertEqual(served["runtimeId"], "mesh-server") + finally: + server.stop() + + def test_a_reset_reported_by_a_receive_is_idle_not_the_end_of_the_thread(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_recvfrom = socket.socket.recvfrom + resets = [3] + try: + y, _ = self._client(server, "peer-y") + + def recvfrom(sock: socket.socket, *args: Any) -> Any: + if sock is server._rudp_socket and resets[0] > 0: + resets[0] -= 1 + raise ConnectionResetError(errno.ECONNRESET, "injected reset") + return original_recvfrom(sock, *args) + + with patch.object(socket.socket, "recvfrom", recvfrom): + self._wait_for(lambda: resets[0] == 0) + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y)) + finally: + server.stop() + + def test_a_permanent_send_failure_ends_only_that_peers_session_and_the_goodbye_names_it(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + failures = [1] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr) and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # The server's response to X can never be sent as built. + x.send("schema", hello(runtime_id="probe").to_bytes()) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertIsNotNone(x.disconnect_reason, "X is told its session ended") + self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: ")) + self.assertGreater(len(x.disconnect_reason), len(b"packet could not be sent: ")) + self.assertTrue(server._rudp_thread.is_alive()) + + # Y is unaffected. + self.assertFalse(x.connected) + self.assertIsNotNone(self._served(y)) + self.assertEqual(server.rudp_send_failures, 0) + finally: + server.stop() + + def test_healthy_sends_are_not_failures(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + x, _ = self._client(server, "peer-x") + self.assertIsNotNone(self._served(x)) + self.assertEqual(server.rudp_send_failures, 0) + finally: + server.stop() + def _wait_for(self, predicate: Callable[[], bool]) -> None: deadline = time.monotonic() + 2.0 while time.monotonic() < deadline: diff --git a/packages/cultnet-py/src/cultnet_py/__init__.py b/packages/cultnet-py/src/cultnet_py/__init__.py index b1734f25..59693f29 100644 --- a/packages/cultnet-py/src/cultnet_py/__init__.py +++ b/packages/cultnet-py/src/cultnet_py/__init__.py @@ -122,6 +122,7 @@ from .transport import ( TcpFramedTransportConnection, compute_reconnect_delay_ms, create_reconnect_policy, + is_permanent_send_error, create_rudp_transport_profile, create_tcp_framed_transport_profile, decode_rudp_packet, @@ -208,6 +209,7 @@ __all__ = [ "CultNetRudpReceiveResult", "CultNetRudpSendOptions", "CultNetRudpSession", + "is_permanent_send_error", "CultNetRudpSessionOptions", "CultNetRudpSocketMode", "CultNetRudpReconnectLoop", diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 355faa51..a0b17cd6 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import secrets import socket import threading @@ -435,6 +436,32 @@ def create_rudp_transport_profile( } +UNSENDABLE_PACKET_REASON = "packet could not be sent" + +_PERMANENT_SEND_ERRNOS = frozenset( + code + for code in ( + errno.EMSGSIZE, + errno.EINVAL, + errno.EAFNOSUPPORT, + getattr(errno, "WSAEMSGSIZE", None), + getattr(errno, "WSAEINVAL", None), + getattr(errno, "WSAEAFNOSUPPORT", None), + ) + if code is not None +) + + +def is_permanent_send_error(error: OSError) -> bool: + """Whether a failed `sendto` can never succeed for the datagram as built, so resending + it is pointless: it is too large for a datagram (EMSGSIZE), its address is malformed for + the socket (EINVAL), or belongs to another address family (EAFNOSUPPORT), with their + Winsock equivalents. Everything else (no route, full buffers, firewall drops, a refusal + reported for an earlier datagram, an interrupted call) is a property of the path or the + moment and may pass, so it is a lost datagram, not a reason to end a session.""" + return error.errno in _PERMANENT_SEND_ERRNOS + + class CultNetRudpSession: RELIABLE_SEND_WINDOW_PACKETS = 32 RECEIVED_SEQUENCE_WINDOW = 4_096 @@ -833,6 +860,14 @@ class CultNetRudpSession: def create_ping(self, payload: bytes = b"") -> CultNetRudpPacket: return self._create_packet(CultNetRudpPacketType.PING, "control", payload) + def end_unsendable_session(self, error: OSError) -> CultNetRudpPacket: + """Ends a session that owes its peer a packet that can never be sent as built + (see `is_permanent_send_error`). Resending it would fail forever, and dropping it + would leave a reliable sequence the peer waits on for good, so the session cannot + be kept. Returns the goodbye for the peer; its reason names the error.""" + self.reset_peer_state() + return self.create_disconnect(f"{UNSENDABLE_PACKET_REASON}: {error}".encode("utf-8")) + def create_disconnect(self, reason: bytes = b"") -> CultNetRudpPacket: self._end_session() return self._create_packet(CultNetRudpPacketType.DISCONNECT, "control", reason) diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index fb2eff54..41665f46 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -877,6 +877,28 @@ class CultNetTests(unittest.TestCase): self.assertEqual(old_ack.ack, oldest_sequence) self.assertEqual(old_ack.ack_mask, 0) + def test_cultnet_rudp_only_a_datagram_that_can_never_be_sent_is_a_permanent_failure(self) -> None: + import errno + + from cultnet_py import is_permanent_send_error + + for code in (errno.EMSGSIZE, errno.EINVAL, errno.EAFNOSUPPORT): + self.assertTrue(is_permanent_send_error(OSError(code, "permanent")), errno.errorcode[code]) + for code in ( + errno.EPERM, + errno.EINTR, + errno.EAGAIN, + errno.EACCES, + errno.ENETDOWN, + errno.ENETUNREACH, + errno.EADDRNOTAVAIL, + errno.ENOBUFS, + errno.ECONNREFUSED, + errno.EHOSTUNREACH, + ): + self.assertFalse(is_permanent_send_error(OSError(code, "transient")), errno.errorcode[code]) + self.assertFalse(is_permanent_send_error(OSError("no errno"))) + def test_cultnet_rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames(self) -> None: server_socket = bind_udp_socket() client_socket = bind_udp_socket() From ac5c86033be44d5a03a76fb940041f8ed288ccbd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:25:04 +0200 Subject: [PATCH 09/33] Ack Cut D D2: pin the disconnect, the resend loop, the Python resend, ack and accept paths The injected permanent failure now takes precedence over the injected transient one, so a test can end a session on a peer that is also losing datagrams and count exactly which datagrams were attempted afterwards. New tests: a hub disconnect that can never be sent returns the error; a peer or session ended in the resend loop is sent nothing more (Rust hub, document server, C#); the goodbye reaches the peer; the Python server ends the session on a permanent failure in a resend, an ack and an accept. The oversized-send test compares the outstanding count with the one the hub's own unacknowledged Accept already holds. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-py/tests/test_cultmesh.py | 100 ++++++++++++++++++ .../cultmesh-rs/src/rudp_document_server.rs | 6 +- .../send_failure_tests.rs | 22 ++++ packages/cultnet-rs/src/rudp.rs | 6 +- .../cultnet-rs/src/rudp/send_failure_tests.rs | 49 ++++++++- src/GameCult.Networking/CultNetTransport.cs | 4 +- .../CultNetRudpSendFailureTests.cs | 22 ++++ 7 files changed, 199 insertions(+), 10 deletions(-) diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 18243923..22a3f692 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -4651,6 +4651,106 @@ class CultMeshRudpSendFailureTests(unittest.TestCase): finally: server.stop() + def test_a_permanent_failure_in_a_resend_ends_only_that_peers_session(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + mode = ["transient"] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr) and mode[0] == "transient": + raise OSError(errno.ENOBUFS, "injected transient failure") + if args and tuple(args[-1]) == tuple(x_addr) and mode[0] == "permanent": + mode[0] = "healthy" + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # X's response is lost, so a resend is pending; then it can never be sent as built. + x.send("schema", hello(runtime_id="probe").to_bytes()) + self._wait_for(lambda: server.rudp_send_failures > 0) + self.assertFalse(x.disconnect_reason, "a transient failure ends no session") + mode[0] = "permanent" + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertIsNotNone(x.disconnect_reason) + self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: ")) + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y)) + finally: + server.stop() + + def test_a_permanent_failure_acknowledging_a_frame_ends_the_session(self) -> None: + import msgpack # type: ignore + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + failures = [1] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr) and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # Not a message the server answers, so its ack is the only datagram owed. + x.send("schema", msgpack.packb(1)) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertIsNotNone(x.disconnect_reason) + self.assertTrue(x.disconnect_reason.startswith(b"packet could not be sent: ")) + self.assertTrue(server._rudp_thread.is_alive()) + finally: + server.stop() + + def test_a_connect_whose_accept_can_never_be_sent_starts_no_session(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + y, _ = self._client(server, "peer-y") + z_socket = bind_udp_socket() + z_addr = z_socket.getsockname()[:2] + failures = [1] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(z_addr) and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + z = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="peer-z", + socket=z_socket, + mode=CultNetRudpSocketMode.CLIENT, + connection_id=server.rudp_connection_id, + remote_addr=("127.0.0.1", server.port), + resend_delay_ms=25, + ) + ) + z.connect(b"join") + # A session that started would resend its Accept, and the resend would get through. + time.sleep(0.5) + z_socket.settimeout(0.05) + with self.assertRaises(TimeoutError): + z_socket.recvfrom(65535) + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y)) + finally: + server.stop() + def _wait_for(self, predicate: Callable[[], bool]) -> None: deadline = time.monotonic() + 2.0 while time.monotonic() < deadline: diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 518a44eb..c0cd200f 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -724,9 +724,6 @@ where fn send_datagram(&mut self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { #[cfg(test)] { - if self.failing_peers.contains(&remote_addr) { - return Err(std::io::Error::other("injected send failure")); - } match self.unsendable_after.get(&remote_addr).copied() { Some(0) => { self.unsendable_after.remove(&remote_addr); @@ -737,6 +734,9 @@ where } None => {} } + if self.failing_peers.contains(&remote_addr) { + return Err(std::io::Error::other("injected send failure")); + } } self.socket.send_to(wire, remote_addr) } diff --git a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs index 3554ab3c..7320c2bf 100644 --- a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs +++ b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs @@ -344,3 +344,25 @@ fn healthy_sends_are_not_failures() { assert_eq!(server.send_failures(), 0); assert_eq!(server.session_count(), 2); } + +#[test] +fn a_session_ended_in_the_resend_loop_is_sent_nothing_more_from_it() { + let (mut server, clock, _received, (mut x, x_addr), (_y, _)) = two_peers(); + server.failing_peers.insert(x_addr); + send(&mut x, &snapshot("one")); + send(&mut x, &snapshot("two")); + for _ in 0..10 { + server.poll_once().unwrap(); + } + let before = server.send_failures(); + + server.unsendable_after.insert(x_addr, 0); + clock.0.store(1_200, Ordering::SeqCst); + let maintenance = server.maintain().unwrap(); + + // Both responses are due. The first can never be sent and ends the session; + // the goodbye is the only other datagram attempted at X. + assert_eq!(maintenance.packets_resent, 2); + assert_eq!(server.send_failures() - before, 1); + assert_eq!(server.session_count(), 1); +} diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 1873c1fb..a9263d06 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -2032,9 +2032,6 @@ impl CultNetRudpServerHub { fn send_datagram(&mut self, wire: &[u8], remote_addr: SocketAddr) -> std::io::Result { #[cfg(test)] { - if self.failing_peers.contains(&remote_addr) { - return Err(std::io::Error::other("injected send failure")); - } match self.unsendable_after.get(&remote_addr).copied() { Some(0) => { self.unsendable_after.remove(&remote_addr); @@ -2045,6 +2042,9 @@ impl CultNetRudpServerHub { } None => {} } + if self.failing_peers.contains(&remote_addr) { + return Err(std::io::Error::other("injected send failure")); + } } self.socket.send_to(wire, remote_addr) } diff --git a/packages/cultnet-rs/src/rudp/send_failure_tests.rs b/packages/cultnet-rs/src/rudp/send_failure_tests.rs index 1c2cb3f2..49a80f80 100644 --- a/packages/cultnet-rs/src/rudp/send_failure_tests.rs +++ b/packages/cultnet-rs/src/rudp/send_failure_tests.rs @@ -162,6 +162,10 @@ fn one_peer( #[test] fn an_oversized_send_returns_the_error_and_queues_nothing() { let (mut hub, mut peer, session) = one_peer(None); + // The hub's own Accept is still awaiting its ack; the refused send adds nothing to it. + let outstanding = hub.peers[&session.remote_addr] + .session + .outstanding_reliable_packet_count(); let error = hub .send(&session, "schema", vec![7; 70_000]) @@ -176,7 +180,7 @@ fn an_oversized_send_returns_the_error_and_queues_nothing() { let remote = session.remote_addr; assert_eq!( hub.peers[&remote].session.outstanding_reliable_packet_count(), - 0, + outstanding, "nothing is pending or queued to resend" ); @@ -213,7 +217,7 @@ fn a_send_that_fails_permanently_after_a_fragment_left_ends_the_session() { #[test] fn a_permanent_failure_in_resends_ends_only_that_peers_session() { - let (mut hub, (_x, x_session), (mut y, y_session)) = two_peers(); + let (mut hub, (mut x, x_session), (mut y, y_session)) = two_peers(); // Both peers' first datagram is lost, so both have a reliable resend due. hub.failing_peers.insert(x_session.remote_addr); hub.failing_peers.insert(y_session.remote_addr); @@ -240,6 +244,16 @@ fn a_permanent_failure_in_resends_ends_only_that_peers_session() { } } assert_eq!(delivered.expect("Y receives its resend").payload, b"for-y"); + // The goodbye reaches X and names the error. + for _ in 0..20 { + x.receive_once().unwrap(); + if x.disconnect_reason().is_some() { + break; + } + } + assert!(is_unsendable_reason( + x.disconnect_reason().expect("X is told its session ended") + )); } #[test] @@ -369,3 +383,34 @@ fn only_a_datagram_that_can_never_be_sent_is_a_permanent_failure() { } } } + +#[test] +fn a_permanent_failure_sending_a_disconnect_is_returned_to_the_caller() { + let (mut hub, _peer, session) = one_peer(None); + hub.unsendable_after.insert(session.remote_addr, 0); + + let error = hub + .disconnect(&session, b"bye".to_vec()) + .expect_err("the goodbye can never be sent"); + assert!(is_permanent_send_error( + error.downcast_ref::().unwrap() + )); +} + +#[test] +fn a_peer_ended_in_the_resend_loop_is_sent_nothing_more_from_it() { + let (mut hub, (_x, x_session), (_y, _y_session)) = two_peers(); + hub.failing_peers.insert(x_session.remote_addr); + hub.send(&x_session, "schema", b"one".to_vec()).unwrap(); + hub.send(&x_session, "schema", b"two".to_vec()).unwrap(); + assert_eq!(hub.stats().send_failures, 2); + + hub.unsendable_after.insert(x_session.remote_addr, 0); + std::thread::sleep(Duration::from_millis(40)); + hub.poll_resends().unwrap(); + + // Both resends are due. The first can never be sent and ends the session; + // the goodbye is the only other datagram attempted at X. + assert_eq!(hub.stats().send_failures, 3); + assert_eq!(hub.sessions().len(), 1); +} diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index f646a7a2..710227be 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -3116,8 +3116,6 @@ namespace GameCult.Networking var wire = CultNetRudpPacketCodec.Encode(packet); try { - if (FailingSendPeers.Contains(remoteEndPoint)) - throw new SocketException((int)SocketError.HostUnreachable); if (UnsendableAfter.TryGetValue(remoteEndPoint, out var remaining)) { if (remaining > 0) @@ -3130,6 +3128,8 @@ namespace GameCult.Networking throw new SocketException((int)SocketError.MessageSize); } } + if (FailingSendPeers.Contains(remoteEndPoint)) + throw new SocketException((int)SocketError.HostUnreachable); var sent = _socket.SendTo(wire, remoteEndPoint); _stats.BytesSent += sent; } diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index ba6cba1f..b43ce5fa 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -293,6 +293,28 @@ namespace GameCult.Networking.Tests Assert.That(server.Stats.PacketsDropped, Is.EqualTo(1)); } + [Test] + public void APeerEndedInTheResendLoopIsSentNothingMoreFromIt() + { + var (server, _, x, _) = TwoPeers(); + using (server) + { + server.FailingSendPeers.Add(x.EndPoint); + server.SendSchema(x.Server, "one"); + server.SendSchema(x.Server, "two"); + Assert.That(server.Stats.SendFailures, Is.EqualTo(2)); + + server.UnsendableAfter[x.EndPoint] = 0; + Thread.Sleep(30); + server.PollResends(); + + // Both resends are due. The first can never be sent and ends the session; the + // goodbye is the only other datagram attempted at X. + Assert.That(server.Stats.SendFailures, Is.EqualTo(3)); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + } + } + [Test] public void AHealthySendCountsItsBytesAndNoFailure() { From 69acee38e7af8ae2e9e3ac9b1e7b56381b3d48ba Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:27:33 +0200 Subject: [PATCH 10/33] Ack Cut D D2 (Python): a refused Accept leaves the peer with the goodbye only The peer is told its session ended, and no Accept resend follows it. Co-Authored-By: Claude Sonnet 5.5 --- packages/cultmesh-py/tests/test_cultmesh.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 22a3f692..86d274ef 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -48,10 +48,12 @@ from cultnet_py import ( CultNetSimulationConsensusCandidate, CultNetSimulationObservation, CultNetSimulationObservationHub, + CultNetRudpPacketType, CultNetRudpSocketMode, CultNetRudpSocketTransportConnection, CultNetRudpSocketTransportOptions, apply_shard_log_response, + decode_rudp_packet, document_delete, document_put_raw, hello, @@ -4744,8 +4746,14 @@ class CultMeshRudpSendFailureTests(unittest.TestCase): # A session that started would resend its Accept, and the resend would get through. time.sleep(0.5) z_socket.settimeout(0.05) - with self.assertRaises(TimeoutError): - z_socket.recvfrom(65535) + received = [] + while True: + try: + wire, _ = z_socket.recvfrom(65535) + except TimeoutError: + break + received.append(decode_rudp_packet(wire).packet_type) + self.assertEqual(received, [CultNetRudpPacketType.DISCONNECT]) self.assertTrue(server._rudp_thread.is_alive()) self.assertIsNotNone(self._served(y)) finally: From 590744f6f1af870b25ab086f9be6462aa78b4cec Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:30:25 +0200 Subject: [PATCH 11/33] Ack Cut D D2 (C#): the oversized-send test compares the outstanding count with the one the listener's own unacknowledged Accept holds Co-Authored-By: Claude Sonnet 5.5 --- .../CultNetRudpSendFailureTests.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index b43ce5fa..e79b319c 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -156,12 +156,15 @@ namespace GameCult.Networking.Tests var (server, _, x, y) = TwoPeers(); using (server) { + // Each peer's own Accept is still awaiting its ack; the refused send adds nothing to it. + var outstanding = x.Server.Session.OutstandingReliablePacketCount; + var otherOutstanding = y.Server.Session.OutstandingReliablePacketCount; var error = Assert.Throws(() => server.SendSchema(x.Server, new byte[70_000]))!; Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True, error.SocketErrorCode.ToString()); Assert.That(server.Stats.SendFailures, Is.Zero); Assert.That(server.Stats.FramesSent, Is.Zero); Assert.That(server.Peers, Has.Count.EqualTo(2), "the caller's error ends no session"); - Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.Zero, "nothing is pending or queued to resend"); + Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing is pending or queued to resend"); // The refused send consumed no sequence: the next frame is delivered in order instead // of waiting behind a gap that never fills. @@ -169,7 +172,7 @@ namespace GameCult.Networking.Tests var data = Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Data); var delivered = x.Session.Receive(data, 1).Delivered; Assert.That(delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)), Is.EqualTo(new[] { "next" })); - Assert.That(y.Server.Session.OutstandingReliablePacketCount, Is.Zero); + Assert.That(y.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(otherOutstanding)); } } From aab72f3d47dbaba2ad8dd6738116b8e354e1a0ec Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 12:49:46 +0200 Subject: [PATCH 12/33] Ack Cut D D2: a refused send withdraws the fragments the flow window queued behind the failed one Rust hub and C# listener: 1,100 one-byte fragments, the first can never be sent, and nothing of the send stays queued. Co-Authored-By: Claude Sonnet 5.5 --- .../cultnet-rs/src/rudp/send_failure_tests.rs | 22 +++++++++++++++++++ .../CultNetRudpSendFailureTests.cs | 17 ++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/packages/cultnet-rs/src/rudp/send_failure_tests.rs b/packages/cultnet-rs/src/rudp/send_failure_tests.rs index 49a80f80..4e5b7125 100644 --- a/packages/cultnet-rs/src/rudp/send_failure_tests.rs +++ b/packages/cultnet-rs/src/rudp/send_failure_tests.rs @@ -414,3 +414,25 @@ fn a_peer_ended_in_the_resend_loop_is_sent_nothing_more_from_it() { assert_eq!(hub.stats().send_failures, 3); assert_eq!(hub.sessions().len(), 1); } + +#[test] +fn a_refused_send_withdraws_the_fragments_the_window_queued_as_well() { + let (mut hub, _peer, session) = one_peer(Some(1)); + let outstanding = hub.peers[&session.remote_addr] + .session + .outstanding_reliable_packet_count(); + hub.unsendable_after.insert(session.remote_addr, 0); + + // 1,100 one-byte fragments: the flow window admits the first ones and + // queues the rest, and the first datagram can never be sent. + hub.send(&session, "schema", vec![7; 1_100]) + .expect_err("the first fragment can never be sent"); + + assert_eq!( + hub.peers[&session.remote_addr] + .session + .outstanding_reliable_packet_count(), + outstanding, + "nothing of the refused send stays queued" + ); +} diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index e79b319c..a3976375 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -176,6 +176,23 @@ namespace GameCult.Networking.Tests } } + [Test] + public void ARefusedSendWithdrawsTheFragmentsTheWindowQueuedAsWell() + { + var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1); + using (server) + { + var outstanding = x.Server.Session.OutstandingReliablePacketCount; + server.UnsendableAfter[x.EndPoint] = 0; + + // 1,100 one-byte fragments: the flow window admits the first ones and queues the rest, + // and the first datagram can never be sent. + Assert.Throws(() => server.SendSchema(x.Server, new byte[1_100])); + + Assert.That(x.Server.Session.OutstandingReliablePacketCount, Is.EqualTo(outstanding), "nothing of the refused send stays queued"); + } + } + [Test] public void ASendThatFailsPermanentlyAfterAFragmentLeftEndsTheSession() { From a44e0199ac4377afe451ff6e0be2dcdfe23a27b0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:03:01 +0200 Subject: [PATCH 13/33] Ack Cut D D2 (cultnet-ts): a send Node refuses never escapes a timer or a receive handler Node reports a failed UDP send two ways: it throws for an argument it refuses (port 0, which any spoofer can claim as a source) and passes a system failure to the callback later. sendRudpDatagram routes both to one failure handler, and isPermanentSendError classifies with the permanent set the other runtimes use (EMSGSIZE, EINVAL, EAFNOSUPPORT, and Node's own ERR_SOCKET_BAD_PORT for port 0). - The socket transport's receive and resend paths hand a failure to that handler: a datagram that can never be sent ends the session (goodbye names the error code, never its message); any other failure is emitted on "error" as the socket always reported it. Caller-directed sends still throw to the caller. A resend loop stops at the generation it ended. - The operation server starts no session for a Connect whose Accept can never be sent (counted as dropped), ends a stored peer whose datagram can never be sent, and counts other failures in sendFailures. - CultNetRudpSession.endUnsendableSession, as in the other runtimes. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/src/index.ts | 2 + packages/cultnet-ts/src/operation-service.ts | 75 ++++-- packages/cultnet-ts/src/rudp.ts | 140 ++++++++++- .../cultnet-ts/test/rudp-send-failure.test.ts | 231 ++++++++++++++++++ 4 files changed, 419 insertions(+), 29 deletions(-) create mode 100644 packages/cultnet-ts/test/rudp-send-failure.test.ts diff --git a/packages/cultnet-ts/src/index.ts b/packages/cultnet-ts/src/index.ts index 8950f638..6b37087d 100644 --- a/packages/cultnet-ts/src/index.ts +++ b/packages/cultnet-ts/src/index.ts @@ -106,6 +106,8 @@ export { createRudpTransportProfile, decodeRudpPacket, encodeRudpPacket, + isPermanentSendError, + sendRudpDatagram, CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS, CultNetRudpReconnectLoop, CultNetRudpSession, diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index 606cd418..ceebaedc 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -12,6 +12,8 @@ import { CultNetRudpSocketTransportConnection, decodeRudpPacket, encodeRudpPacket, + isPermanentSendError, + sendRudpDatagram, type CultNetRudpPacket, } from "./rudp"; import { CultNetPeer } from "./peer"; @@ -38,6 +40,12 @@ export interface CultNetOperationServer { readonly endpoint: string; /** Datagrams read and discarded: malformed, unadmitted, refused by their session, or failed in handling. */ readonly packetsDropped: number; + /** + * Datagrams that could not be sent to a peer and may yet be (no route, full buffers): + * each is that peer's lost datagram, resent if it was reliable. One that can never be + * sent as built ends that peer's session instead and is not counted here. + */ + readonly sendFailures: number; close(): Promise; } @@ -55,8 +63,12 @@ interface RemoteSession { handling: number; /** When the last handler finished: the response still needs its acknowledgement. */ lastHandledAtMs: number; + /** A packet it owed could never be sent, so the session ended. */ + unsendable: boolean; } +type SendPacket = (key: string, peer: RemoteSession, packet: CultNetRudpPacket) => void; + export async function startCultNetOperationServer( options: CultNetOperationServerOptions, ): Promise { @@ -65,11 +77,23 @@ export async function startCultNetOperationServer( const socket = createSocket("udp4"); const connectionId = options.connectionId ?? DEFAULT_CONNECTION_ID; const sessions = new Map(); - const sendPacket = (remote: RemoteInfo, packet: CultNetRudpPacket): void => { - const wire = encodeRudpPacket(packet); - socket.send(wire, remote.port, remote.address); - }; let packetsDropped = 0; + let sendFailures = 0; + const sendPacket: SendPacket = (key, peer, packet) => { + sendRudpDatagram(socket, encodeRudpPacket(packet), peer.remote.port, peer.remote.address, (error) => { + if (!isPermanentSendError(error)) { + sendFailures += 1; + return; + } + if (peer.unsendable) return; + // The datagram can never be sent as built: the session that owes it ends, never + // the server, and the peer is told. + peer.unsendable = true; + if (sessions.get(key) === peer) sessions.delete(key); + const goodbye = encodeRudpPacket(peer.session.endUnsendableSession(error)); + sendRudpDatagram(socket, goodbye, peer.remote.port, peer.remote.address, () => {}); + }); + }; socket.on("message", (wire, remote) => { // What a datagram carries is the sender's business: a rejection here must // drop and count the packet, never become an unhandled rejection that ends @@ -89,8 +113,8 @@ export async function startCultNetOperationServer( if (peer.handling > 0 || now - peer.lastHandledAtMs < idleTimeoutMs) continue; if (peer.session.checkTimeout(now, idleTimeoutMs)) sessions.delete(key); } - for (const peer of sessions.values()) { - for (const packet of peer.session.dueResends(Date.now())) sendPacket(peer.remote, packet); + for (const [key, peer] of sessions) { + for (const packet of peer.session.dueResends(Date.now())) sendPacket(key, peer, packet); } }, 25); resendTimer.unref?.(); @@ -99,6 +123,7 @@ export async function startCultNetOperationServer( return { endpoint, get packetsDropped() { return packetsDropped; }, + get sendFailures() { return sendFailures; }, close: async () => { clearInterval(resendTimer); await closeSocket(socket); @@ -153,7 +178,7 @@ async function handleServerDatagram( options: CultNetOperationServerOptions, wire: Buffer, remote: RemoteInfo, - sendPacket: (remote: RemoteInfo, packet: CultNetRudpPacket) => void, + sendPacket: SendPacket, ): Promise { let packet: CultNetRudpPacket; try { @@ -165,14 +190,24 @@ async function handleServerDatagram( const key = `${remote.address}:${remote.port}`; let peer = sessions.get(key); if (packet.packetType === "connect") { - if (!peer) { - if (sessions.size >= MAX_OPERATION_SESSIONS) return false; - peer = { session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), remote, handling: 0, lastHandledAtMs: 0 }; - sessions.set(key, peer); + if (peer) { + // A Connect from an admitted peer repeats: the session answers with the + // Accept already owed and queues nothing. + sendPacket(key, peer, peer.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); + return true; } - // A Connect from an admitted peer repeats: the session answers with the - // Accept already owed and queues nothing. - sendPacket(remote, peer.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); + if (sessions.size >= MAX_OPERATION_SESSIONS) return false; + const candidate: RemoteSession = { + session: new CultNetRudpSession({ connectionId, resendDelayMs: 25 }), + remote, + handling: 0, + lastHandledAtMs: 0, + unsendable: false, + }; + sendPacket(key, candidate, candidate.session.acceptConnect(packet, Date.now(), encode("cultnet-operation-service"))); + // A peer whose Accept can never be sent cannot be answered, so no session starts. + if (candidate.unsendable) return false; + sessions.set(key, candidate); return true; } if (!peer) return false; @@ -183,8 +218,8 @@ async function handleServerDatagram( endSession(sessions, key, peer, sendPacket); return false; } - if (result.reply) sendPacket(remote, result.reply); - for (const ready of result.readyToSend ?? []) sendPacket(remote, ready); + if (result.reply) sendPacket(key, peer, result.reply); + for (const ready of result.readyToSend ?? []) sendPacket(key, peer, ready); if (result.disconnected) { sessions.delete(key); return true; @@ -208,7 +243,7 @@ async function handleServerDatagram( ordered: true, nowMs: Date.now(), maxFragmentBytes: options.maxFragmentBytes ?? 2048, - })) sendPacket(remote, responsePacket); + })) sendPacket(key, peer, responsePacket); } catch { // The session recorded this request's sequence, so keeping it would // acknowledge the retransmit of a request that was never handled. @@ -217,7 +252,7 @@ async function handleServerDatagram( } } if (packet.packetType === "data" || result.delivered.length > 0) { - sendPacket(remote, peer.session.createAckForReceived(packet.sequence)); + sendPacket(key, peer, peer.session.createAckForReceived(packet.sequence)); } return true; } @@ -231,11 +266,11 @@ function endSession( sessions: Map, key: string, peer: RemoteSession, - sendPacket: (remote: RemoteInfo, packet: CultNetRudpPacket) => void, + sendPacket: SendPacket, ): void { if (sessions.get(key) === peer) sessions.delete(key); peer.session.resetPeerState(); - sendPacket(peer.remote, peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + sendPacket(key, peer, peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); } function parseRudpEndpoint(endpoint: string): { host: string; port: number } { diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index e32bb2d7..4b166d95 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -645,6 +645,18 @@ export class CultNetRudpSession { }); } + /// Ends a session that owes its peer a packet that can never be sent as + /// built (see `isPermanentSendError`). Resending it would fail forever, and + /// dropping it would leave a reliable sequence the peer waits on for good, so + /// the session cannot be kept. Returns the goodbye for the peer; its reason + /// names the error's code, never its message, which can quote the address. + endUnsendableSession(error: Error): CultNetRudpPacket { + this.resetPeerState(); + return this.createDisconnect( + Buffer.from(`${RUDP_UNSENDABLE_PACKET_REASON}: ${sendErrorCode(error)}`, "utf8"), + ); + } + checkTimeout(nowMs: number, timeoutMs: number): boolean { if (!this.#connected || this.#lastReceivedAtMs === undefined) { return false; @@ -1100,11 +1112,22 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement // The goodbye is built after the reset, or its ack field would acknowledge // the very frame the session refused. this.#session.resetPeerState(); + this.#endWithGoodbye(this.#session.createDisconnect(reason)); + } + + /** The session owes its peer a packet that can never be sent as built. */ + #endUnsendableSession(error: Error): void { + this.#endWithGoodbye(this.#session.endUnsendableSession(error)); + } + + #endWithGoodbye(goodbye: CultNetRudpPacket): void { + const reason = goodbye.payload; this.#endedReason = reason; - try { - this.#sendPacket(this.#session.createDisconnect(reason)); - } catch { + if (this.#remoteHost && this.#remotePort !== undefined) { // Best-effort: the session ends whether or not the peer hears it. + const wire = encodeRudpPacket(goodbye); + this.#stats.bytesSent += wire.length; + sendRudpDatagram(this.#socket, wire, this.#remotePort, this.#remoteHost, () => {}); } if (this.#mode === "server") { // Only a Connect can claim the endpoint again. @@ -1175,7 +1198,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement this.#stats.packetsDropped += 1; return; } - this.#sendPacket(accept); + this.#sendInPoll(accept); return; } @@ -1192,10 +1215,10 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } try { if (result.reply) { - this.#sendPacket(result.reply); + this.#sendInPoll(result.reply); } for (const ready of result.readyToSend ?? []) { - this.#sendPacket(ready); + this.#sendInPoll(ready); } if (result.pong) { this.emit("pong", { payload: result.pongPayload ?? new Uint8Array() }); @@ -1214,7 +1237,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement return; } if (packet.packetType === "accept" || packet.packetType === "data" || result.delivered.length > 0) { - this.#sendPacket(this.#session.createAckForReceived(packet.sequence)); + this.#sendInPoll(this.#session.createAckForReceived(packet.sequence)); } } catch (error) { this.emit("error", error instanceof Error ? error : new Error(String(error))); @@ -1222,18 +1245,62 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } #sendDueResends(): void { + const generation = this.#session.generation; for (const packet of this.#session.dueResends(Date.now())) { - this.#sendPacket(packet); + this.#sendInPoll(packet); + // A resend that can never be sent ended the session and what it owed. + if (this.#session.generation !== generation) return; } } + /** A caller-directed send: a failure the send throws is the caller's error. */ #sendPacket(packet: CultNetRudpPacket): void { + const thrown = this.#transmit(packet); + if (thrown) throw thrown; + } + + /** + * A send with no caller to hand a failure to (a receive or a resend): the failure + * is handled as every later one is, and never escapes the timer or the socket's + * "message" handler. + */ + #sendInPoll(packet: CultNetRudpPacket): void { + const thrown = this.#transmit(packet); + if (thrown) this.#sendFailed(thrown); + } + + /** Sends one packet and returns the failure the send threw, if any. */ + #transmit(packet: CultNetRudpPacket): Error | undefined { if (!this.#remoteHost || this.#remotePort === undefined) { throw new Error("RUDP socket transport does not have a remote endpoint."); } const wire = encodeRudpPacket(packet); this.#stats.bytesSent += wire.length; - this.#socket.send(wire, this.#remotePort, this.#remoteHost); + let thrown: Error | undefined; + let sending = true; + const generation = this.#session.generation; + sendRudpDatagram(this.#socket, wire, this.#remotePort, this.#remoteHost, (error) => { + if (sending) { + thrown = error; + } else if (this.#session.generation === generation) { + // A later failure belongs to the session generation that sent it. + this.#sendFailed(error); + } + }); + sending = false; + return thrown; + } + + /** + * A datagram that can never be sent ends the session; any other failure is a lost + * datagram, reported on "error" as the socket reports it. + */ + #sendFailed(error: Error): void { + if (isPermanentSendError(error)) { + this.#endUnsendableSession(error); + return; + } + this.emit("error", error); } } @@ -1327,6 +1394,61 @@ export class CultNetRudpReconnectLoop extends EventEmitter { } } +/** Prefix of the goodbye reason for a session that owed a packet which can never be sent. */ +const RUDP_UNSENDABLE_PACKET_REASON = "packet could not be sent"; + +/** + * Failures that can never pass for the datagram as built: too large for a datagram + * (`EMSGSIZE`), an address malformed for the socket (`EINVAL`, and Node's own + * `ERR_SOCKET_BAD_PORT`, which it throws for port 0 before the system would say + * `EINVAL`), or an address of another family (`EAFNOSUPPORT`). libuv names these the + * same on every platform. + */ +const PERMANENT_SEND_ERROR_CODES: ReadonlySet = new Set([ + "EMSGSIZE", + "EINVAL", + "EAFNOSUPPORT", + "ERR_SOCKET_BAD_PORT", +]); + +function sendErrorCode(error: Error): string { + const code = (error as NodeJS.ErrnoException).code; + return typeof code === "string" && code.length > 0 ? code : "UNKNOWN"; +} + +/** + * Whether a failed UDP send can never succeed for the datagram as built, so resending it is + * pointless and the session that owes it cannot be kept. Everything else (no route, full + * buffers, a closed socket, an interrupted call) is a property of the path or the moment: a + * lost datagram, not a reason to end a session. + */ +export function isPermanentSendError(error: Error): boolean { + return PERMANENT_SEND_ERROR_CODES.has(sendErrorCode(error)); +} + +/** + * Sends one datagram. Node reports a failed send two ways: it throws for an argument it + * refuses (port 0 among them) and passes a system failure to the callback later. Both + * reach `onFailure` and nothing else, so a datagram that cannot be sent to one peer never + * escapes a timer or a receive handler, and never reaches the socket's "error" event. A + * failure the send throws reaches `onFailure` before this returns. + */ +export function sendRudpDatagram( + socket: Socket, + wire: Uint8Array, + port: number, + address: string, + onFailure: (error: Error) => void, +): void { + try { + socket.send(wire, port, address, (error) => { + if (error) onFailure(error); + }); + } catch (error) { + onFailure(error instanceof Error ? error : new Error(String(error))); + } +} + export function createRudpTransportProfile( runtimeId: string, options: RudpTransportProfileOptions = {}, diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts new file mode 100644 index 00000000..9abfe3d8 --- /dev/null +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -0,0 +1,231 @@ +// A datagram that cannot be sent to one peer is that peer's problem alone. Node throws +// for a send it refuses outright (a peer at UDP port 0, which any spoofer can claim) +// and reports a system failure later through the callback; neither may escape a +// timer or a receive handler, and only a datagram that can never be sent as built +// ends the session that owes it. +import assert from "node:assert/strict"; +import dgram, { type RemoteInfo, type Socket } from "node:dgram"; +import test from "node:test"; +import { + CultNetRudpSession, + CultNetRudpSocketTransportConnection, + encodeRudpPacket, + isPermanentSendError, + startCultNetOperationServer, + type CultNetOperationResponseMessage, +} from "../src"; + +async function bindUdpSocket(): Promise { + const socket = dgram.createSocket("udp4"); + await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); + return socket; +} + +function udpPort(socket: Socket): number { + const address = socket.address(); + assert.notEqual(typeof address, "string"); + return address.port; +} + +async function waitFor(predicate: () => boolean, description: string): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > 2_000) { + throw new Error(`Timed out waiting for ${description}.`); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** A Connect as it arrives from UDP source port 0: Node refuses to send anything back. */ +function connectFromPortZero(socket: Socket, connectionId: number): void { + const wire = Buffer.from(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(Date.now()))); + const remote: RemoteInfo = { address: "127.0.0.1", family: "IPv4", port: 0, size: wire.length }; + socket.emit("message", wire, remote); +} + +/** Records every exception that would otherwise end the process. */ +function watchUncaught(): { errors: unknown[]; stop: () => void } { + const errors: unknown[] = []; + const listener = (error: unknown) => errors.push(error); + process.on("uncaughtException", listener); + return { errors, stop: () => process.off("uncaughtException", listener) }; +} + +test("only a datagram that can never be sent as built is a permanent send failure", () => { + const failure = (code: string) => Object.assign(new Error("send failed"), { code }); + for (const code of ["EMSGSIZE", "EINVAL", "EAFNOSUPPORT", "ERR_SOCKET_BAD_PORT"]) { + assert.equal(isPermanentSendError(failure(code)), true, code); + } + for (const code of ["EHOSTUNREACH", "ENETUNREACH", "ECONNREFUSED", "ENOBUFS", "EAGAIN", "ERR_SOCKET_DGRAM_NOT_RUNNING"]) { + assert.equal(isPermanentSendError(failure(code)), false, code); + } + assert.equal(isPermanentSendError(new Error("no code")), false); +}); + +test("a server transport answering a Connect from port 0 ends that session and keeps serving", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203090; + const uncaught = watchUncaught(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + const errors: Error[] = []; + const reasons: string[] = []; + server.on("error", (error: Error) => errors.push(error)); + server.on("disconnect", ({ reason }: { reason: Uint8Array }) => reasons.push(Buffer.from(reason).toString("utf8"))); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + try { + connectFromPortZero(serverSocket, connectionId); + // Many resend ticks: nothing the port-0 session owed is ever retried. + await sleep(100); + // The goodbye names the error by its code, never by a message quoting the address. + assert.deepEqual(reasons, ["packet could not be sent: ERR_SOCKET_BAD_PORT"]); + + const frames: string[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(Buffer.from(frame.payload).toString("utf8"))); + client.connect(); + await waitFor(() => client.connected && server.connected, "a real client's handshake"); + client.send("schema", Buffer.from("after", "utf8")); + await waitFor(() => frames.length === 1, "the real client's frame"); + assert.deepEqual(frames, ["after"]); + assert.deepEqual(errors, []); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.close(); + server.close(); + } +}); + +test("an operation server answering a Connect from port 0 starts no session and keeps serving", async () => { + const connectionId = 0x43554c54; + const uncaught = watchUncaught(); + // The server owns its socket; this captures it so a datagram can arrive from port 0. + const createSocket = dgram.createSocket; + let serverSocket: Socket | undefined; + Object.assign(dgram, { + createSocket: (...args: Parameters) => { + serverSocket = createSocket(...args); + return serverSocket; + }, + }); + const server = await startCultNetOperationServer({ + runtimeId: "operations", + handler: (request): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: request.payloadSchema, + payloadEncoding: request.payloadEncoding, + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "operations", + }), + }).finally(() => Object.assign(dgram, { createSocket })); + const raw = await bindUdpSocket(); + const port = Number(new URL(server.endpoint).port); + const accepts: number[] = []; + raw.on("message", () => accepts.push(Date.now())); + try { + connectFromPortZero(serverSocket!, connectionId); + // The resend timer runs every 25 ms; a session that had started would owe its Accept. + await sleep(150); + assert.equal(server.packetsDropped, 1, "the Connect started no session"); + assert.equal(server.sendFailures, 0); + + // A real peer is still answered. + raw.send(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(0)), port, "127.0.0.1"); + await waitFor(() => accepts.length > 0, "the real peer's Accept"); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + raw.close(); + await server.close(); + } +}); + +test("a server transport whose resend can never be sent ends the session and sends nothing more of it", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203094; + const uncaught = watchUncaught(); + const clientPort = udpPort(clientSocket); + // Sends to the client fail as the test says: later through the callback, or at once. + let mode: "healthy" | "lost" | "never" = "healthy"; + const send = serverSocket.send.bind(serverSocket) as (...args: unknown[]) => void; + Object.assign(serverSocket, { + send: (message: Uint8Array, port: number, address: string, callback?: (error: Error | null) => void) => { + if (port !== clientPort || mode === "healthy") { + send(message, port, address, callback); + } else if (mode === "lost") { + process.nextTick(() => callback?.(Object.assign(new Error("lost"), { code: "EHOSTUNREACH" }))); + } else { + throw Object.assign(new Error("never"), { code: "EMSGSIZE" }); + } + }, + }); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + const errors: string[] = []; + const reasons: string[] = []; + server.on("error", (error: NodeJS.ErrnoException) => errors.push(error.code ?? "")); + server.on("disconnect", ({ reason }: { reason: Uint8Array }) => reasons.push(Buffer.from(reason).toString("utf8"))); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + resendDelayMs: 10, + resendPollMs: 5, + }); + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); + mode = "lost"; + server.send("schema", Buffer.from("one", "utf8")); + server.send("schema", Buffer.from("two", "utf8")); + await waitFor(() => errors.length >= 2, "both writes lost"); + assert.deepEqual([...new Set(errors)], ["EHOSTUNREACH"], "a loss that may pass is reported and ends nothing"); + assert.deepEqual(reasons, []); + + // Both writes are due again together; the first can never be sent. + mode = "never"; + await waitFor(() => reasons.length > 0, "the session to end"); + await sleep(50); + assert.deepEqual(reasons, ["packet could not be sent: EMSGSIZE"]); + assert.equal(server.connected, false); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.close(); + server.close(); + } +}); From cc750d4e4cfa9d45a41de88de9db9aecab676865 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:03:05 +0200 Subject: [PATCH 14/33] Ack Cut D D2 (cultmesh-ts): the document server's sends are guarded, classified, and a refused packet ends only its peer Every socket send in the RUDP document server goes through sendRudpDatagram. A Connect from UDP port 0 used to store a session whose pending Accept threw from the resend interval on every tick, an uncaughtException that ends the process. Now: - a Connect whose Accept can never be sent starts no session and leaves a session already at that endpoint as it was; - a datagram that can never be sent as built ends only that peer's session (goodbye, onSessionClosed); any other failure is a lost datagram, reported through onError, and ends nothing; - a packet the session refuses (bad fragment metadata) ends that peer's session with "session refused a packet" instead of leaving a session that has recorded a sequence it never delivered. Co-Authored-By: Claude Opus 5.5 --- packages/cultmesh-ts/src/index.ts | 121 ++++++---- .../rudp-document-server-send-failure.test.ts | 209 ++++++++++++++++++ 2 files changed, 286 insertions(+), 44 deletions(-) create mode 100644 packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index b89cd823..212e3836 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -21,7 +21,9 @@ import { defineCultNetDocumentBinding, encodeCultNetMessageForWire, encodeRudpPacket, + isPermanentSendError, parseCultNetMessage, + sendRudpDatagram, type CultNetErrorMessage, type CultNetDocumentBinding, type CultNetMessage, @@ -29,6 +31,7 @@ import { type CultNetOperationResponseMessage, type CultNetRawDocumentRecord, type CultNetReconnectPolicy, + type CultNetRudpPacket, type CultNetSchemaCatalogOptions, type CultNetSnapshotResponseRawMessage, type CultNetWireContract, @@ -5761,14 +5764,17 @@ export class CultMesh { const port = options.bindPort ?? 0; const bind = { host, port }; const socket = options.socket ?? createSocket(host.includes(":") ? "udp6" : "udp4"); - const sessions = new Map; - }>(); + }; + const sessions = new Map(); let sessionSequence = 0; const resendPollMs = Math.max(10, options.resendPollMs ?? 25); const sessionTimeoutMs = Math.max(1_000, options.sessionTimeoutMs ?? 30_000); @@ -5794,19 +5800,15 @@ export class CultMesh { if (packet.packetType === "connect") { const connectPayload = Uint8Array.from(packet.payload ?? []); if (record?.session.connectRepeats(packet)) { - socket.send( - encodeRudpPacket(record.session.acceptConnect(packet, nowMs)), - remote.port, - remote.address, - ); + sendPacket(record, record.session.acceptConnect(packet, nowMs)); return; } - if (record) closeSession(record); - record = { + const next: SessionRecord = { sessionId: `${runtimeId}:${++sessionSequence}`, remote: { address: remote.address, family: remote.family, port: remote.port }, connectPayload, closed: false, + unsendable: false, session: new CultNetRudpSession({ connectionId, initialSequence: options.initialSequence, @@ -5815,8 +5817,12 @@ export class CultMesh { }), work: Promise.resolve(), }; - sessions.set(key, record); - socket.send(encodeRudpPacket(record.session.acceptConnect(packet, nowMs)), remote.port, remote.address); + sendPacket(next, next.session.acceptConnect(packet, nowMs)); + // A peer whose Accept can never be sent cannot be answered: no session + // starts, and one already there is left as it was. + if (next.unsendable) return; + if (record) closeSession(record); + sessions.set(key, next); return; } @@ -5824,12 +5830,22 @@ export class CultMesh { return; } - const result = record.session.receive(packet, nowMs); + let result: ReturnType; + try { + result = record.session.receive(packet, nowMs); + } catch { + // receive() has already recorded the packet's reliable sequence, so the + // session cannot be kept: a retransmit would be acknowledged and the frame + // silently lost. It ends, never the server, and the peer is told. What a + // datagram carries is the sender's business, so nothing is reported. + endRefusedSession(key, record); + return; + } if (result.reply) { - socket.send(encodeRudpPacket(result.reply), record.remote.port, record.remote.address); + sendPacket(record, result.reply); } for (const ready of result.readyToSend ?? []) { - socket.send(encodeRudpPacket(ready), record.remote.port, record.remote.address); + sendPacket(record, ready); } for (const frame of result.delivered) { if (frame.channelId !== "schema") { @@ -5845,11 +5861,7 @@ export class CultMesh { return; } if (packet.reliable) { - socket.send( - encodeRudpPacket(record.session.createAckForReceived(packet.sequence)), - record.remote.port, - record.remote.address, - ); + sendPacket(record, record.session.createAckForReceived(packet.sequence)); } } catch (error) { reportError(error); @@ -5858,7 +5870,7 @@ export class CultMesh { socket.on("error", reportError); async function handleDocumentServerFrame( - record: { sessionId: string; remote: { address: string; family: string; port: number }; session: CultNetRudpSession; connectPayload: Uint8Array; closed: boolean }, + record: SessionRecord, payload: Uint8Array, ): Promise { if (record.closed) return; @@ -5920,13 +5932,7 @@ export class CultMesh { } } - function publicSession(record: { - sessionId: string; - remote: { address: string; family: string; port: number }; - session: CultNetRudpSession; - connectPayload: Uint8Array; - closed: boolean; - }): CultMeshRudpServerSession { + function publicSession(record: SessionRecord): CultMeshRudpServerSession { return { sessionId: record.sessionId, remote: { ...record.remote }, @@ -5938,31 +5944,58 @@ export class CultMesh { }; } - function notifySessionClosed(record: { - sessionId: string; - remote: { address: string; family: string; port: number }; - session: CultNetRudpSession; - connectPayload: Uint8Array; - closed: boolean; - }): void { + function notifySessionClosed(record: SessionRecord): void { if (!options.onSessionClosed) return; void Promise.resolve(options.onSessionClosed(publicSession(record))).catch(reportError); } - function closeSession(record: { - sessionId: string; - remote: { address: string; family: string; port: number }; - session: CultNetRudpSession; - connectPayload: Uint8Array; - closed: boolean; - }): void { + function closeSession(record: SessionRecord): void { if (record.closed) return; record.closed = true; notifySessionClosed(record); } + /** + * Sends one packet to a session's peer. A failure is that peer's alone: one that may + * pass (no route, full buffers) is a lost datagram, reported, and resent if it was + * reliable; one that can never pass as built ends that peer's session. Neither ends + * the server, a timer or another peer's session. + */ + function sendPacket(record: SessionRecord, packet: CultNetRudpPacket): void { + sendRudpDatagram(socket, encodeRudpPacket(packet), record.remote.port, record.remote.address, (error) => { + if (isPermanentSendError(error)) { + endUnsendableSession(record, error); + } else { + reportError(error); + } + }); + } + + function endRefusedSession(key: string, record: SessionRecord): void { + sessions.delete(key); + closeSession(record); + // The goodbye is built after the reset, or its ack field would acknowledge the + // very frame the session refused. + record.session.resetPeerState(); + const goodbye = encodeRudpPacket(record.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + sendRudpDatagram(socket, goodbye, record.remote.port, record.remote.address, () => {}); + } + + /** The session owes its peer a packet that can never be sent as built. */ + function endUnsendableSession(record: SessionRecord, error: Error): void { + if (record.unsendable) return; + record.unsendable = true; + const key = `${record.remote.address}:${record.remote.port}`; + // A session that never started, or was already replaced, has nothing to end. + if (sessions.get(key) !== record) return; + sessions.delete(key); + closeSession(record); + const goodbye = encodeRudpPacket(record.session.endUnsendableSession(error)); + sendRudpDatagram(socket, goodbye, record.remote.port, record.remote.address, () => {}); + } + function sendSchemaMessage( - record: { sessionId: string; remote: { address: string; port: number }; session: CultNetRudpSession; closed: boolean }, + record: SessionRecord, message: CultNetMessage, ): void { if (record.closed) throw new Error(`CultMesh RUDP session ${record.sessionId} is closed.`); @@ -5972,7 +6005,7 @@ export class CultMesh { ordered: true, nowMs: Date.now(), })) { - socket.send(encodeRudpPacket(packet), record.remote.port, record.remote.address); + sendPacket(record, packet); } } @@ -5999,7 +6032,7 @@ export class CultMesh { continue; } for (const packet of record.session.dueResends(nowMs)) { - socket.send(encodeRudpPacket(packet), record.remote.port, record.remote.address); + sendPacket(record, packet); } } }, resendPollMs); diff --git a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts new file mode 100644 index 00000000..8fa0f6a4 --- /dev/null +++ b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts @@ -0,0 +1,209 @@ +// A datagram the document server cannot send to one peer is that peer's problem +// alone. It never escapes a timer or the receive handler, never stops the server +// serving the others, and only a datagram that can never be sent as built ends +// the session that owes it. +import assert from "node:assert/strict"; +import { createSocket, type RemoteInfo, type Socket } from "node:dgram"; +import test from "node:test"; +import { encode } from "@msgpack/msgpack"; +import { + CultNetDocumentRegistry, + CultNetRudpSession, + decodeRudpPacket, + encodeRudpPacket, + type CultNetPeer, +} from "cultnet-ts"; +import { CultMesh } from "../src/index"; + +async function waitFor(predicate: () => boolean, description: string): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > 3_000) { + throw new Error(`Timed out waiting for ${description}.`); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +function watchUncaught(): { errors: unknown[]; stop: () => void } { + const errors: unknown[] = []; + const listener = (error: unknown) => errors.push(error); + process.on("uncaughtException", listener); + return { errors, stop: () => process.off("uncaughtException", listener) }; +} + +function put(recordKey: string) { + return { + schemaVersion: "cultnet.document_put_raw.v0" as const, + messageId: `put-${recordKey}`, + document: { + schemaId: "cultmesh.note.v0", + recordKey, + storedAt: new Date().toISOString(), + payloadEncoding: "messagepack" as const, + payload: encode(recordKey), + }, + }; +} + +interface Server { + socket: Socket; + server: ReturnType; + /** The record key of every put received, with the port it came from. */ + puts: Map; + closed: number[]; + errors: Error[]; + peer(name: string): Promise; +} + +async function startServer(connectionId: number, socket: Socket): Promise { + const puts = new Map(); + const closed: number[] = []; + const errors: Error[] = []; + const server = CultMesh.createRudpDocumentServer("send-failure-server", connectionId, { + documents: new CultNetDocumentRegistry(), + socket, + bindHost: "127.0.0.1", + bindPort: 0, + resendDelayMs: 10, + resendPollMs: 10, + sessionTimeoutMs: 1_000, + onDocumentPutRaw: (document) => { + puts.set(document.recordKey, document.remote.port); + }, + onSessionClosed: (session) => { + closed.push(session.remote.port); + }, + onError: (error) => errors.push(error), + }); + await server.start(); + const peer = (name: string) => + CultMesh.createRudpPeer(name, connectionId, `rudp://127.0.0.1:${server.bind.port}`, { + resendDelayMs: 10, + resendPollMs: 5, + connectTimeoutMs: 2_000, + }); + return { socket, server, puts, closed, errors, peer }; +} + +test("a Connect whose Accept can never be sent starts no session, and the server keeps serving", async () => { + const connectionId = 0x10203091; + const uncaught = watchUncaught(); + const socket = createSocket("udp4"); + const served = await startServer(connectionId, socket); + let peer: CultNetPeer | undefined; + try { + // A Connect as it arrives from UDP source port 0, which Node refuses to answer. + const wire = Buffer.from(encodeRudpPacket(new CultNetRudpSession({ connectionId }).createConnect(Date.now()))); + const remote: RemoteInfo = { address: "127.0.0.1", family: "IPv4", port: 0, size: wire.length }; + socket.emit("message", wire, remote); + // Past the idle timeout: a session that had started would have been closed by now. + await sleep(1_300); + + peer = await served.peer("after-port-zero"); + peer.send(put("after")); + await waitFor(() => served.puts.has("after"), "the real peer's put"); + assert.deepEqual(served.closed, [], "no session started for port 0"); + assert.deepEqual(served.errors.map((error) => error.message), []); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + peer?.close(); + served.server.close(); + } +}); + +test("a datagram that may yet pass is reported; one that never can ends only that peer's session", async () => { + const connectionId = 0x10203092; + const socket = createSocket("udp4"); + // Sends to a failing port report the failure the way Node reports a system one: + // later, through the send's callback. + const failing = new Map(); + const send = socket.send.bind(socket) as (...args: unknown[]) => void; + Object.assign(socket, { + send: (message: Uint8Array, port: number, address: string, callback?: (error: Error | null) => void) => { + const code = failing.get(port); + if (code === undefined) { + send(message, port, address, callback); + return; + } + process.nextTick(() => callback?.(Object.assign(new Error(`send failed: ${code}`), { code }))); + }, + }); + const served = await startServer(connectionId, socket); + let x: CultNetPeer | undefined; + let y: CultNetPeer | undefined; + try { + x = await served.peer("x"); + y = await served.peer("y"); + x.send(put("x-1")); + await waitFor(() => served.puts.has("x-1"), "X's first put"); + const xPort = served.puts.get("x-1")!; + + // No route: X's acknowledgements are lost datagrams, reported, and end nothing. + failing.set(xPort, "EHOSTUNREACH"); + x.send(put("x-2")); + await waitFor(() => served.errors.some((error) => (error as NodeJS.ErrnoException).code === "EHOSTUNREACH"), "the reported loss"); + y.send(put("y-1")); + await waitFor(() => served.puts.has("y-1"), "Y's put"); + assert.deepEqual(served.closed, [], "a loss that may pass ends no session"); + + // X's retransmitted put is acknowledged again, and that datagram can never be sent. + failing.set(xPort, "EMSGSIZE"); + await waitFor(() => served.closed.length > 0, "X's session to end"); + assert.deepEqual(served.closed, [xPort]); + assert.equal( + served.errors.some((error) => (error as NodeJS.ErrnoException).code === "EMSGSIZE"), + false, + "a datagram that can never be sent ends the session instead of being reported", + ); + + y.send(put("y-2")); + await waitFor(() => served.puts.has("y-2"), "Y's put after X ended"); + assert.deepEqual(served.closed, [xPort]); + } finally { + x?.close(); + y?.close(); + served.server.close(); + } +}); + +test("a packet the session refuses ends only that peer's session, and the peer is told", async () => { + const connectionId = 0x10203093; + const socket = createSocket("udp4"); + const served = await startServer(connectionId, socket); + const raw = createSocket("udp4"); + await new Promise((resolve) => raw.bind(0, "127.0.0.1", resolve)); + const arrived: ReturnType[] = []; + raw.on("message", (wire) => arrived.push(decodeRudpPacket(wire))); + let y: CultNetPeer | undefined; + try { + const session = new CultNetRudpSession({ connectionId }); + raw.send(encodeRudpPacket(session.createConnect(Date.now())), served.server.bind.port, "127.0.0.1"); + await waitFor(() => arrived.some((packet) => packet.packetType === "accept"), "the Accept"); + session.receive(arrived.find((packet) => packet.packetType === "accept")!, Date.now()); + y = await served.peer("y"); + + // A fragment whose metadata the session refuses: fragment id 0 names no set. + const [fragment] = session.sendMany("schema", new Uint8Array([1]), { reliable: false }); + raw.send( + encodeRudpPacket({ ...fragment!, fragmentId: 0, fragmentIndex: 0, fragmentCount: 2 }), + served.server.bind.port, + "127.0.0.1", + ); + await waitFor(() => arrived.some((packet) => packet.packetType === "disconnect"), "the goodbye"); + const goodbye = arrived.find((packet) => packet.packetType === "disconnect")!; + assert.equal(Buffer.from(goodbye.payload ?? []).toString("utf8"), "session refused a packet"); + assert.deepEqual(served.closed, [(raw.address() as { port: number }).port]); + + y.send(put("y-after")); + await waitFor(() => served.puts.has("y-after"), "Y's put"); + assert.deepEqual(served.errors.map((error) => error.message), []); + } finally { + y?.close(); + raw.close(); + served.server.close(); + } +}); From 06f8a31cb561e25158c0341f63cf122b9e60b949 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:03:11 +0200 Subject: [PATCH 15/33] Ack Cut D (Python): a packet one peer's session refuses ends that session, never the RUDP thread session.receive raises ValueError on bad fragment metadata, and nothing in _rudp_loop caught it, so one admitted peer could end the thread for every peer. The server now ends that peer's session with "session refused a packet", as the socket transport and the Rust servers do. Tests pin that, and two D2 rules that were unpinned: a resend loop sends nothing more to a peer it just ended (P1), and responses after one that can never be sent are not sent (P3). Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-py/src/cultmesh_py/server.py | 22 ++- packages/cultmesh-py/tests/test_cultmesh.py | 125 ++++++++++++++++++ 2 files changed, 146 insertions(+), 1 deletion(-) diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index 41d9d9e9..be8cd903 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -262,7 +262,14 @@ class CultMeshLocalServer: if peer is None: continue - result = peer.session.receive(packet, now_ms) + try: + result = peer.session.receive(packet, now_ms) + except ValueError: + # receive() has already recorded the packet's reliable sequence, so the + # session cannot be kept: a retransmit would be acknowledged and the + # frame silently lost. It ends, never the thread, and the peer is told. + self._end_refused_peer(rudp_socket, peers, remote_addr) + continue unsendable: OSError | None = None for outgoing in (*([result.reply] if result.reply is not None else []), *result.ready_to_send): unsendable = self._send_rudp_packet(rudp_socket, remote_addr, outgoing) @@ -349,6 +356,19 @@ class CultMeshLocalServer: self._rudp_send_failures += 1 return None + def _end_refused_peer( + self, + rudp_socket: socket.socket, + peers: dict[tuple[str, int], _RudpPeerConnection], + remote_addr: tuple[str, int], + ) -> None: + peer = peers.pop(remote_addr) + # The goodbye is built after the reset, or its ack field would acknowledge the + # very frame the session refused. + peer.session.reset_peer_state() + goodbye = peer.session.create_disconnect(b"session refused a packet") + self._send_rudp_packet(rudp_socket, remote_addr, goodbye) + def _end_unsendable_peer( self, rudp_socket: socket.socket, diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 86d274ef..6428a9d9 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -4759,6 +4759,131 @@ class CultMeshRudpSendFailureTests(unittest.TestCase): finally: server.stop() + def test_a_packet_the_session_refuses_ends_only_that_peers_session(self) -> None: + from cultnet_py import CultNetRudpPacket, encode_rudp_packet + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + x, _ = self._client(server, "peer-x") + y, _ = self._client(server, "peer-y") + # A fragment whose metadata the session refuses: fragment id 0 names no set. + refused = CultNetRudpPacket( + packet_type=CultNetRudpPacketType.DATA, + connection_id=server.rudp_connection_id, + sequence=0, + ack=0, + ack_mask=0, + channel_id="schema", + reliable=False, + fragment_id=0, + fragment_index=0, + fragment_count=2, + payload=b"x", + ) + x.socket.sendto(encode_rudp_packet(refused), ("127.0.0.1", server.port)) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertEqual(x.disconnect_reason, b"session refused a packet") + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y), "Y is still served") + finally: + server.stop() + + def _raw_peer(self, server: Any) -> tuple[socket.socket, Any]: + """A peer driven packet by packet, to deliver frames in an order a client would not.""" + from cultnet_py import CultNetRudpSession, CultNetRudpSessionOptions, encode_rudp_packet + + sock = bind_udp_socket() + sock.settimeout(0.05) + session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=server.rudp_connection_id)) + sock.sendto(encode_rudp_packet(session.create_connect(0, b"join")), ("127.0.0.1", server.port)) + wire, _ = sock.recvfrom(65535) + accept = decode_rudp_packet(wire) + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + session.receive(accept, 0) + return sock, session + + def test_responses_after_one_that_can_never_be_sent_are_not_sent(self) -> None: + from cultnet_py import CultNetRudpSendOptions, encode_rudp_packet + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + sock, session = self._raw_peer(server) + addr = sock.getsockname()[:2] + request = hello(runtime_id="probe").to_bytes() + options = CultNetRudpSendOptions(reliable=True, ordered=True) + first = session.send("schema", request, options) + second = session.send("schema", request, options) + attempts: list[CultNetRudpPacketType] = [] + failures = [1] + + def sendto(sock_: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(addr): + packet_type = decode_rudp_packet(data).packet_type + attempts.append(packet_type) + if packet_type == CultNetRudpPacketType.DATA and failures[0] > 0: + failures[0] -= 1 + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock_, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + # The second frame waits for the first, so one receive delivers both, and + # the first of the two responses can never be sent as built. + sock.sendto(encode_rudp_packet(second), ("127.0.0.1", server.port)) + time.sleep(0.1) + attempts.clear() + sock.sendto(encode_rudp_packet(first), ("127.0.0.1", server.port)) + self._wait_for(lambda: CultNetRudpPacketType.DISCONNECT in attempts) + time.sleep(0.1) + + # The unsendable response, then only the goodbye: no second response, no ack. + self.assertEqual(attempts, [CultNetRudpPacketType.DATA, CultNetRudpPacketType.DISCONNECT]) + self.assertTrue(server._rudp_thread.is_alive()) + finally: + server.stop() + + def test_a_peer_ended_in_the_resend_loop_is_sent_nothing_more_from_it(self) -> None: + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + original_sendto = socket.socket.sendto + try: + x, x_addr = self._client(server, "peer-x") + mode = ["transient"] + attempts: list[CultNetRudpPacketType] = [] + + def sendto(sock: socket.socket, data: bytes, *args: Any) -> int: + if args and tuple(args[-1]) == tuple(x_addr): + packet_type = decode_rudp_packet(data).packet_type + if mode[0] != "transient": + attempts.append(packet_type) + # Only responses fail; X's requests are acknowledged, so X goes quiet. + if packet_type == CultNetRudpPacketType.DATA and mode[0] == "transient": + raise OSError(errno.ENOBUFS, "injected transient failure") + if packet_type == CultNetRudpPacketType.DATA and mode[0] == "permanent": + mode[0] = "healthy" + raise OSError(errno.EMSGSIZE, "Message too long") + return original_sendto(sock, data, *args) + + with patch.object(socket.socket, "sendto", sendto): + x.send("schema", hello(runtime_id="one").to_bytes()) + x.send("schema", hello(runtime_id="two").to_bytes()) + for _ in range(40): + x.receive_once() + time.sleep(0.005) + self._wait_for(lambda: server.rudp_send_failures >= 2) + # Both responses are owed; the next resend of the first can never be sent. + mode[0] = "permanent" + self._wait_for(lambda: CultNetRudpPacketType.DISCONNECT in attempts) + time.sleep(0.2) + + self.assertEqual(attempts, [CultNetRudpPacketType.DATA, CultNetRudpPacketType.DISCONNECT]) + self.assertTrue(server._rudp_thread.is_alive()) + finally: + server.stop() + def _wait_for(self, predicate: Callable[[], bool]) -> None: deadline = time.monotonic() + 2.0 while time.monotonic() < deadline: From b96420b606bbac019cb0d201f88b249b6ca684ab Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:03:16 +0200 Subject: [PATCH 16/33] Ack Cut D (Rust): pin the receive reset, the ack after an unsendable reply, and a refusal at every fragment - Document server: a reset reported by a receive is idle, not an error (Windows only, where the system reports it; run on Starfire). - Document server: no ack follows a reply that can never be sent. The guard is not decoration: without it the ack's success overwrote the failure and the session that owed the Pong was kept. - Hub: a send refused at any fragment index delivers nothing of it; at index 0 the next send arrives whole (Soul's probe, committed). Co-Authored-By: Claude Opus 5.5 --- .../send_failure_tests.rs | 78 +++++++++++++++++++ .../cultnet-rs/src/rudp/send_failure_tests.rs | 64 +++++++++++++++ 2 files changed, 142 insertions(+) diff --git a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs index 7320c2bf..68351b35 100644 --- a/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs +++ b/packages/cultmesh-rs/src/rudp_document_server/send_failure_tests.rs @@ -366,3 +366,81 @@ fn a_session_ended_in_the_resend_loop_is_sent_nothing_more_from_it() { assert_eq!(server.send_failures() - before, 1); assert_eq!(server.session_count(), 1); } + +#[test] +fn no_ack_follows_a_reply_that_can_never_be_sent() { + let (mut server, _clock, _received, _x, _y) = two_peers(); + let target = server.local_addr().unwrap(); + // A peer driven packet by packet: a client never sends a reliable Ping, but a + // peer may, and the server both answers it and acknowledges it. + let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(20))) + .unwrap(); + let addr = socket.local_addr().unwrap(); + let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 7, + ..Default::default() + }); + let connect = session.create_connect(0, Vec::new()).unwrap(); + socket + .send_to(&encode_rudp_packet(&connect).unwrap(), target) + .unwrap(); + let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES]; + for _ in 0..10 { + server.poll_once().unwrap(); + } + let (read, _) = socket.recv_from(&mut wire).unwrap(); + let accept = decode_rudp_packet(&wire[..read]).unwrap(); + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + session.receive(&accept, 0).unwrap(); + assert_eq!(server.session_count(), 3); + + let mut ping = session.create_ping(b"p".to_vec()); + ping.reliable = true; + ping.sequence = connect.sequence.wrapping_add(1); + // The Pong can never be sent; the ack after it could. + server.unsendable_after.insert(addr, 0); + socket + .send_to(&encode_rudp_packet(&ping).unwrap(), target) + .unwrap(); + for _ in 0..10 { + server.poll_once().unwrap(); + } + + // The session ends over the Pong: the peer gets its goodbye and nothing else. + assert_eq!(server.session_count(), 2); + let mut arrived = Vec::new(); + while let Ok((read, _)) = socket.recv_from(&mut wire) { + arrived.push(decode_rudp_packet(&wire[..read]).unwrap()); + } + assert_eq!(arrived.len(), 1, "{arrived:?}"); + assert_eq!(arrived[0].packet_type, CultNetRudpPacketType::Disconnect); + assert!(is_unsendable_reason(&arrived[0].payload)); +} + +/// Windows reports an ICMP port-unreachable for an earlier datagram as +/// `ConnectionReset` on the next receive. It names no datagram of this poll, so +/// the poll is idle and the server goes on serving. Only Windows reports it, so +/// only a Windows run can see the rule break. +#[cfg(windows)] +#[test] +fn a_reset_reported_by_a_receive_is_idle_not_an_error() { + let (mut server, _clock, received, (mut x, _), (mut y, _)) = two_peers(); + // X asks for a snapshot and goes away before the answer. + send(&mut x, &snapshot("x-snapshot")); + drop(x); + for _ in 0..20 { + server + .poll_once() + .expect("a reset from X's closed port is not an error"); + thread::sleep(Duration::from_millis(1)); + } + + send(&mut y, &put("y-put")); + for _ in 0..20 { + server.poll_once().unwrap(); + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(received.lock().unwrap().as_slice(), ["y-put"]); +} diff --git a/packages/cultnet-rs/src/rudp/send_failure_tests.rs b/packages/cultnet-rs/src/rudp/send_failure_tests.rs index 4e5b7125..4bff8cf9 100644 --- a/packages/cultnet-rs/src/rudp/send_failure_tests.rs +++ b/packages/cultnet-rs/src/rudp/send_failure_tests.rs @@ -436,3 +436,67 @@ fn a_refused_send_withdraws_the_fragments_the_window_queued_as_well() { "nothing of the refused send stays queued" ); } + +fn receive_frame( + peer: &mut CultNetRudpSocketTransportConnection, + hub: &mut CultNetRudpServerHub, +) -> Option> { + for _ in 0..600 { + if let Some(frame) = peer.receive_once().unwrap() { + return Some(frame.payload); + } + let _ = hub.receive_event_once().unwrap(); + hub.poll_resends().unwrap(); + } + None +} + +/// Wherever in a fragmented send the datagram that can never be sent falls, the +/// peer never delivers a frame built from it. At the first fragment nothing left, +/// the send is withdrawn, and the next send is delivered whole; later, the peer has +/// seen part of it, and the session ends with nothing delivered. +#[test] +fn a_send_refused_at_any_fragment_delivers_nothing_of_it() { + for fragments in [5_usize, 40] { + for failed in 0..fragments { + let (mut hub, mut peer, session) = one_peer(Some(1000)); + hub.send(&session, "schema", b"warm".to_vec()).unwrap(); + assert_eq!(receive_frame(&mut peer, &mut hub).as_deref(), Some(&b"warm"[..])); + let refused: Vec = (0..fragments * 1000).map(|i| (i % 251) as u8).collect(); + hub.unsendable_after.insert(session.remote_addr, failed); + let result = hub.send(&session, "schema", refused); + if hub.unsendable_after.remove(&session.remote_addr).is_some() { + // The flow window queued that fragment rather than sending it now. + assert!(result.is_ok(), "{fragments}/{failed}: not reached, yet refused"); + continue; + } + assert!(result.is_err(), "{fragments}/{failed}"); + if failed == 0 { + assert_eq!(hub.sessions().len(), 1, "{fragments}/{failed}"); + let next: Vec = (0..fragments * 1000).map(|i| ((i % 13) + 100) as u8).collect(); + hub.send(&session, "schema", next.clone()).unwrap(); + let delivered = receive_frame(&mut peer, &mut hub).expect("the next send arrives"); + assert!(delivered == next, "{fragments}/{failed}: delivered something else"); + for _ in 0..30 { + if let Some(frame) = peer.receive_once().unwrap() { + panic!("{fragments}/{failed}: a stray frame of {} bytes", frame.payload.len()); + } + let _ = hub.receive_event_once().unwrap(); + hub.poll_resends().unwrap(); + } + } else { + assert!(hub.sessions().is_empty(), "{fragments}/{failed}"); + for _ in 0..30 { + match peer.receive_once() { + Ok(Some(frame)) => panic!( + "{fragments}/{failed}: the peer delivered {} bytes", + frame.payload.len() + ), + Ok(None) => {} + Err(_) => break, + } + } + } + } + } +} From cce4d391b8e9db9e264c029f2a3acc002b1209df Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:03:22 +0200 Subject: [PATCH 17/33] Ack Cut D D2 (C#): only the polling thread changes the peer set A caller-directed Send that failed after a fragment left called EndUnsendablePeer on the caller's thread, removing from _peers and raising PeerDisconnected while the polling thread could be changing _peers too. Send now ends the session and sends the goodbye under the peer's SessionGate, and queues the peer; TryReceiveOnce and PollResends retire queued peers before anything else. A peer a new Connect already replaced is skipped, so a stale peer object cannot remove the peer that replaced it. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 50 ++++++++++++--- .../CultNetRudpSendFailureTests.cs | 61 +++++++++++++++++++ 2 files changed, 104 insertions(+), 7 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 710227be..4ff0fbab 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -1,5 +1,6 @@ using System; using System.Buffers.Binary; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.IO; @@ -2682,6 +2683,11 @@ namespace GameCult.Networking /// Gets the last transport-level remote disconnect reason, if one was received. /// public byte[]? DisconnectReason { get; internal set; } + + // The goodbye's reason once the session ended over a packet it could never send. Written + // under SessionGate by whichever thread's send found the failure; the polling thread reads + // it when it retires the peer. + internal byte[]? UnsendableReason { get; set; } } /// @@ -2719,6 +2725,10 @@ namespace GameCult.Networking private readonly CultNetTransportStats _stats = new CultNetTransportStats(); private readonly Dictionary _peers = new Dictionary(StringComparer.Ordinal); private readonly Queue _deliveredFrames = new Queue(); + // Peers whose session ended over a packet it could never send, waiting for the polling thread + // to retire them. Only the polling thread changes _peers or raises PeerDisconnected; a send on + // a caller's thread ends the session and leaves the peer here. + private readonly ConcurrentQueue _unsendablePeers = new ConcurrentQueue(); private bool _disposed; /// @@ -2795,8 +2805,9 @@ namespace GameCult.Networking } if (unsendable != null) { + // The polling thread retires the peer; this thread only ends its session. if (failedIndex > 0) - EndUnsendablePeer(peer, unsendable); + EndUnsendableSession(peer, unsendable); throw unsendable; } _stats.FramesSent++; @@ -2843,6 +2854,7 @@ namespace GameCult.Networking /// public bool TryReceiveOnce(out CultNetRudpSocketServerFrame? delivered) { + RetireUnsendablePeers(); if (_deliveredFrames.Count > 0) { delivered = _deliveredFrames.Dequeue(); @@ -3046,6 +3058,7 @@ namespace GameCult.Networking /// public void PollResends() { + RetireUnsendablePeers(); if (_socket.Poll(0, SelectMode.SelectRead)) return; foreach (var peer in _peers.Values.ToArray()) @@ -3082,16 +3095,39 @@ namespace GameCult.Networking // names the error. private void EndUnsendablePeer(CultNetRudpSocketServerPeer peer, SocketException error) { - var key = RemoteKey(peer.RemoteEndPoint); - if (!_peers.TryGetValue(key, out var current) || !ReferenceEquals(current, peer)) - return; - _peers.Remove(key); + EndUnsendableSession(peer, error); + RetireUnsendablePeers(); + } + + // Ends the session and tells the peer, on whichever thread found the failure. The peer stays in + // _peers until the polling thread retires it. + private void EndUnsendableSession(CultNetRudpSocketServerPeer peer, SocketException error) + { CultNetRudpPacket goodbye; lock (peer.SessionGate) + { + if (peer.UnsendableReason != null) + return; goodbye = peer.Session.EndUnsendable(error); - peer.DisconnectReason = goodbye.Payload; + peer.UnsendableReason = goodbye.Payload; + } SendPacket(peer.RemoteEndPoint, goodbye); - PeerDisconnected?.Invoke(peer); + _unsendablePeers.Enqueue(peer); + } + + // Polling thread only. A peer a new Connect already replaced was reported then, and its key now + // names the peer that replaced it. + private void RetireUnsendablePeers() + { + while (_unsendablePeers.TryDequeue(out var peer)) + { + var key = RemoteKey(peer.RemoteEndPoint); + if (!_peers.TryGetValue(key, out var current) || !ReferenceEquals(current, peer)) + continue; + _peers.Remove(key); + peer.DisconnectReason = peer.UnsendableReason; + PeerDisconnected?.Invoke(peer); + } } /// diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index a3976375..25c07e7b 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -204,12 +204,73 @@ namespace GameCult.Networking.Tests var error = Assert.Throws(() => server.SendSchema(x.Server, new byte[3000]))!; Assert.That(CultNetRudpSession.IsPermanentSendError(error), Is.True); + // The send ends the session and tells the peer; it leaves the peer for the polling + // thread to retire, because only that thread changes the peer set. + Assert.That(x.Server.Connected, Is.False); + Assert.That(Drain(x.Socket).Last().PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(ended, Is.Empty); + + server.ReceiveOnce(); Assert.That(server.Peers, Has.Count.EqualTo(1)); Assert.That(ended, Is.EqualTo(new[] { x.Server })); Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); } } + [Test] + public void APeerEndedByASendOnAnotherThreadIsRetiredByThePollingThread() + { + var (server, _, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var retiredOn = new List(); + server.PeerDisconnected += _ => retiredOn.Add(Environment.CurrentManagedThreadId); + server.UnsendableAfter[x.EndPoint] = 1; + + SocketException? thrown = null; + var sender = new Thread(() => + { + try { server.SendSchema(x.Server, new byte[3000]); } + catch (SocketException error) { thrown = error; } + }); + sender.Start(); + sender.Join(); + Assert.That(thrown, Is.Not.Null); + Assert.That(server.Peers, Has.Count.EqualTo(2), "the sending thread leaves the peer set alone"); + + server.PollResends(); + Assert.That(retiredOn, Is.EqualTo(new[] { Environment.CurrentManagedThreadId })); + Assert.That(server.Peers, Has.Count.EqualTo(1)); + } + } + + [Test] + public void ASendOnAPeerAlreadyReplacedLeavesThePeerThatReplacedIt() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var stale = x.Server; + // X connects again as a new session: the listener replaces the peer it held. + var fresh = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(fresh.CreateConnect(0)), listenerEndPoint); + server.ReceiveOnce(); + var replacing = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint)); + Assert.That(replacing, Is.Not.SameAs(stale)); + var ended = RecordDisconnects(server); + + // A caller still holding the old peer sends through it, and that send fails after a + // fragment left. + server.UnsendableAfter[x.EndPoint] = 1; + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + server.ReceiveOnce(); + + Assert.That(server.Peers, Has.Member(replacing)); + Assert.That(server.Peers, Has.Count.EqualTo(2)); + Assert.That(ended, Is.Empty); + } + } + [Test] public void APermanentFailureInResendsEndsOnlyThatPeersSession() { From 1f25195afe9caa10faeda4911682d5432626b46f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:51:47 +0200 Subject: [PATCH 18/33] cultnet-ts, cultmesh-ts: a send the network refuses is a counted lost datagram, never an "error" Node reports a failed send only to the send's callback, never on the socket's "error" event. Emitting it as "error" from the transport ended any process with no listener (invokeCultNetOperation among them) once per resend tick. The transport and the document server now count it in sendFailures, as the operation server, Rust and Python already do. The tests send to 255.255.255.255 with no error listener. Co-Authored-By: Claude Opus 5.5 --- packages/cultmesh-ts/src/index.ts | 14 +++- .../rudp-document-server-send-failure.test.ts | 9 ++- packages/cultnet-ts/src/rudp.ts | 6 +- packages/cultnet-ts/src/transport.ts | 8 +++ .../cultnet-ts/test/rudp-send-failure.test.ts | 66 ++++++++++++++++++- 5 files changed, 94 insertions(+), 9 deletions(-) diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index dd9280e7..2e487c77 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -4310,6 +4310,12 @@ export interface CultMeshRudpServerSession { export interface CultMeshRudpDocumentServer { readonly bind: { host: string; port: number }; + /** + * Datagrams that could not be sent to a peer. Each is that peer's lost datagram: a + * reliable packet stays pending and is resent, and no other peer is affected. A datagram + * that can never be sent as built is not counted here: it ends that peer's session. + */ + readonly sendFailures: number; start(): Promise; close(): void; } @@ -5782,6 +5788,7 @@ export class CultMesh { }; const sessions = new Map(); let sessionSequence = 0; + let sendFailures = 0; const resendPollMs = Math.max(10, options.resendPollMs ?? 25); const sessionTimeoutMs = Math.max(1_000, options.sessionTimeoutMs ?? 30_000); const wireContract = options.wireContract ?? "cultnet.schema.v0"; @@ -5963,7 +5970,7 @@ export class CultMesh { /** * Sends one packet to a session's peer. A failure is that peer's alone: one that may - * pass (no route, full buffers) is a lost datagram, reported, and resent if it was + * pass (no route, full buffers) is a lost datagram, counted, and resent if it was * reliable; one that can never pass as built ends that peer's session. Neither ends * the server, a timer or another peer's session. */ @@ -5972,7 +5979,7 @@ export class CultMesh { if (isPermanentSendError(error)) { endUnsendableSession(record, error); } else { - reportError(error); + sendFailures += 1; } }); } @@ -6017,6 +6024,9 @@ export class CultMesh { return { bind, + get sendFailures() { + return sendFailures; + }, async start(): Promise { await new Promise((resolve, reject) => { socket.once("error", reject); diff --git a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts index 8fa0f6a4..534a9440 100644 --- a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts +++ b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts @@ -115,7 +115,7 @@ test("a Connect whose Accept can never be sent starts no session, and the server } }); -test("a datagram that may yet pass is reported; one that never can ends only that peer's session", async () => { +test("a datagram that may yet pass is counted; one that never can ends only that peer's session", async () => { const connectionId = 0x10203092; const socket = createSocket("udp4"); // Sends to a failing port report the failure the way Node reports a system one: @@ -142,13 +142,16 @@ test("a datagram that may yet pass is reported; one that never can ends only tha await waitFor(() => served.puts.has("x-1"), "X's first put"); const xPort = served.puts.get("x-1")!; - // No route: X's acknowledgements are lost datagrams, reported, and end nothing. + // No route: X's acknowledgements are lost datagrams, counted, never reported as errors, + // and end nothing. + assert.equal(served.server.sendFailures, 0); failing.set(xPort, "EHOSTUNREACH"); x.send(put("x-2")); - await waitFor(() => served.errors.some((error) => (error as NodeJS.ErrnoException).code === "EHOSTUNREACH"), "the reported loss"); + await waitFor(() => served.server.sendFailures >= 2, "the counted losses"); y.send(put("y-1")); await waitFor(() => served.puts.has("y-1"), "Y's put"); assert.deepEqual(served.closed, [], "a loss that may pass ends no session"); + assert.deepEqual(served.errors.map((error) => error.message), [], "a lost datagram is not an error"); // X's retransmitted put is acknowledged again, and that datagram can never be sent. failing.set(xPort, "EMSGSIZE"); diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index d3617471..da431cd9 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -1049,6 +1049,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement framesReceived: 0, framesSent: 0, packetsDropped: 0, + sendFailures: 0, }; constructor(options: CultNetRudpSocketTransportOptions) { @@ -1363,14 +1364,15 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement /** * A datagram that can never be sent ends the session; any other failure is a lost - * datagram, reported on "error" as the socket reports it. + * datagram, counted and never emitted. Node reports a failed send only to its callback, + * not on the socket's "error" event, and an "error" with no listener ends the process. */ #sendFailed(error: Error): void { if (isPermanentSendError(error)) { this.#endUnsendableSession(error); return; } - this.emit("error", error); + this.#stats.sendFailures += 1; } } diff --git a/packages/cultnet-ts/src/transport.ts b/packages/cultnet-ts/src/transport.ts index f900aa23..02f5e8c1 100644 --- a/packages/cultnet-ts/src/transport.ts +++ b/packages/cultnet-ts/src/transport.ts @@ -16,6 +16,13 @@ export interface CultNetTransportStats { framesSent: number; /** Datagrams read and discarded because they belong to no session on this transport. */ packetsDropped: number; + /** + * Datagrams this transport could not send and treats as lost (no route, full buffers): + * a reliable packet stays pending and is resent. A send a caller made throws its failure + * to the caller instead, and a datagram that can never be sent as built ends the session. + * Zero for stream transports. + */ + sendFailures: number; } export interface CultNetReconnectPolicy { @@ -195,6 +202,7 @@ export class TcpFramedTransportConnection extends EventEmitter implements CultNe framesReceived: 0, framesSent: 0, packetsDropped: 0, + sendFailures: 0, }; constructor(stream: Duplex, profile: CultNetTransportProfile) { diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts index 9abfe3d8..edcf0d08 100644 --- a/packages/cultnet-ts/test/rudp-send-failure.test.ts +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -10,6 +10,7 @@ import { CultNetRudpSession, CultNetRudpSocketTransportConnection, encodeRudpPacket, + invokeCultNetOperation, isPermanentSendError, startCultNetOperationServer, type CultNetOperationResponseMessage, @@ -212,8 +213,8 @@ test("a server transport whose resend can never be sent ends the session and sen mode = "lost"; server.send("schema", Buffer.from("one", "utf8")); server.send("schema", Buffer.from("two", "utf8")); - await waitFor(() => errors.length >= 2, "both writes lost"); - assert.deepEqual([...new Set(errors)], ["EHOSTUNREACH"], "a loss that may pass is reported and ends nothing"); + await waitFor(() => server.stats.sendFailures >= 4, "both writes lost, and lost again when resent"); + assert.deepEqual(errors, [], "a loss that may pass is counted, never emitted"); assert.deepEqual(reasons, []); // Both writes are due again together; the first can never be sent. @@ -229,3 +230,64 @@ test("a server transport whose resend can never be sent ends the session and sen server.close(); } }); + +// The kernel refuses a datagram to the limited broadcast address from a socket that has +// not asked for broadcast (EACCES): a real failure that may pass once the socket or the +// route changes, so it is a lost datagram. Node reports it only to the send's callback, +// never on the socket's "error" event, so nothing may turn it into an "error" either. +const UNREACHABLE = { host: "255.255.255.255", port: 9 }; + +test("a client transport with no error listener counts a send the network refuses and does not end the process", async () => { + const socket = await new Promise((resolve) => { + const created = dgram.createSocket("udp4"); + created.bind(0, "0.0.0.0", () => resolve(created)); + }); + const uncaught = watchUncaught(); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket, + mode: "client", + remoteHost: UNREACHABLE.host, + remotePort: UNREACHABLE.port, + connectionId: 0x10203095, + resendDelayMs: 10, + resendPollMs: 5, + }); + try { + client.connect(); + // The Connect and its resends, each refused by the kernel. + await waitFor(() => client.stats.sendFailures >= 5, "the refused Connect and its resends"); + assert.equal(client.connected, false); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.close(); + } +}); + +test("an operation call the network refuses rejects with a timeout and does not end the process", async () => { + const uncaught = watchUncaught(); + try { + await assert.rejects( + invokeCultNetOperation( + `rudp://${UNREACHABLE.host}:${UNREACHABLE.port}`, + { + schemaVersion: "cultnet.operation_request.v0", + messageId: "refused", + serviceId: "service", + operation: "operation", + payloadSchema: "payload", + payloadEncoding: "messagepack", + payload: new Uint8Array(), + diagnostics: [], + sourceRuntimeId: "caller", + }, + { runtimeId: "caller", timeoutMs: 400 }, + ), + /CultNet operation connection timed out/, + ); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + } +}); From e8e8e5bac10b9ac46e15a1a8d6f51e3a4faf189f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:54:33 +0200 Subject: [PATCH 19/33] GameCult.Networking: a Connect that replaces a peer ends the old peer's session under its gate A caller still holding the replaced peer (a schema handler awaiting work for it) could send into the endpoint the new session owns: a goodbye after a partial failure disconnected the new client, a fixed initial sequence delivered the stale frame in place of the real one, and a drawn one stalled the new session. The replaced session now ends under the SessionGate every Send takes, so a stale Send throws and nothing of it leaves. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 20 ++++-- .../CultNetRudpSendFailureTests.cs | 68 +++++++++++++++---- 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index a34043d1..dcfe3b6a 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -2966,14 +2966,20 @@ namespace GameCult.Networking _peers.TryGetValue(peerKey, out var admitted); // The admitted peer's session decides whether this Connect repeats the one it accepted: // answer a repeat with the Accept already owed and queue nothing. Any other Connect is - // a new generation. + // a new generation, and the admitted peer's session ends here, under the gate every + // Send takes: a caller still holding the replaced peer can send nothing more into the + // endpoint the new session now owns. if (admitted != null) { var repeats = false; SocketException? unsendableReply = null; lock (admitted.SessionGate) { - if (admitted.Session.ConnectRepeats(packet)) + if (!admitted.Session.ConnectRepeats(packet)) + { + admitted.Session.ResetPeerState(); + } + else { repeats = true; CultNetRudpPacket reply; @@ -3012,11 +3018,15 @@ namespace GameCult.Networking unsendableAccept = SendPacket(peer.RemoteEndPoint, peer.Session.AcceptConnect(packet, NowMs(), _acceptPayload)); if (unsendableAccept != null) { - // The peer cannot be answered, so no session starts. + // The peer cannot be answered, so no session starts, and the one it replaced has + // already ended. + _peers.Remove(peerKey); _stats.PacketsDropped++; - return true; } - _peers[peerKey] = peer; + else + { + _peers[peerKey] = peer; + } if (admitted != null) { admitted.DisconnectReason = ReplacedPeerReason; diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index 25c07e7b..70f9acc7 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -60,7 +60,7 @@ namespace GameCult.Networking.Tests return peer; } - private static (CultNetRudpSocketTransportServer Server, EndPoint ServerEndPoint, Peer X, Peer Y) TwoPeers(int? maxFragmentBytes = null) + private static (CultNetRudpSocketTransportServer Server, EndPoint ServerEndPoint, Peer X, Peer Y) TwoPeers(int? maxFragmentBytes = null, uint? initialSequence = null) { var serverSocket = Bind(); var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions @@ -69,7 +69,8 @@ namespace GameCult.Networking.Tests Socket = serverSocket, ConnectionId = ConnectionId, ResendDelayMs = 5, - MaxFragmentBytes = maxFragmentBytes + MaxFragmentBytes = maxFragmentBytes, + InitialSequence = initialSequence }); var x = Connect(server, serverSocket.LocalEndPoint!); var y = Connect(server, serverSocket.LocalEndPoint!); @@ -244,33 +245,74 @@ namespace GameCult.Networking.Tests } } + // A client that connects again from the same endpoint replaces the peer the listener held. A + // caller still holding the replaced peer (a handler still running for it, say) reaches nothing + // of the session that replaced it: no data and no goodbye. + + /// X connects again as a new session, and the handshake completes. + private static (CultNetRudpSocketServerPeer Stale, CultNetRudpSocketServerPeer Replacing) Reconnect( + CultNetRudpSocketTransportServer server, EndPoint listenerEndPoint, Peer x) + { + var stale = x.Server; + Drain(x.Socket); + x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint); + server.ReceiveOnce(); + x.Session.Receive(Drain(x.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + x.Server = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint)); + Assert.That(x.Server, Is.Not.SameAs(stale)); + Assert.That(x.Session.Connected, Is.True); + return (stale, x.Server); + } + [Test] - public void ASendOnAPeerAlreadyReplacedLeavesThePeerThatReplacedIt() + public void ASendOnAReplacedPeerThrowsAndNoGoodbyeReachesTheSessionThatReplacedIt() { var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); using (server) { - var stale = x.Server; - // X connects again as a new session: the listener replaces the peer it held. - var fresh = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); - x.Socket.SendTo(CultNetRudpPacketCodec.Encode(fresh.CreateConnect(0)), listenerEndPoint); - server.ReceiveOnce(); - var replacing = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint)); - Assert.That(replacing, Is.Not.SameAs(stale)); + var (stale, replacing) = Reconnect(server, listenerEndPoint, x); var ended = RecordDisconnects(server); + Assert.That(stale.Connected, Is.False, "the replaced peer's session ended with the replacement"); - // A caller still holding the old peer sends through it, and that send fails after a - // fragment left. + // Had the stale send gone out, a failure after its first fragment would end the + // session that sent it and say goodbye to the endpoint. server.UnsendableAfter[x.EndPoint] = 1; - Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); server.ReceiveOnce(); + Assert.That(Drain(x.Socket), Is.Empty, "nothing of the stale send reached the client"); + Assert.That(x.Session.Connected, Is.True); + Assert.That(replacing.Connected, Is.True); Assert.That(server.Peers, Has.Member(replacing)); Assert.That(server.Peers, Has.Count.EqualTo(2)); Assert.That(ended, Is.Empty); } } + // With a fixed initial sequence the stale frame would take the sequence the real one gets, and + // the real one would be dropped as a duplicate. With a drawn one it would land far ahead of + // the new session's window, and nothing after it would be delivered. + [TestCase(1000u)] + [TestCase(null)] + public void ASendOnAReplacedPeerDeliversNothingAndTheReplacingPeersFrameArrives(uint? initialSequence) + { + var (server, listenerEndPoint, x, _) = TwoPeers(initialSequence: initialSequence); + using (server) + { + var (stale, replacing) = Reconnect(server, listenerEndPoint, x); + + Assert.Throws(() => server.SendSchema(stale, "STALE")); + server.SendSchema(replacing, "REAL"); + + var delivered = Drain(x.Socket) + .SelectMany(packet => x.Session.Receive(packet, 1).Delivered) + .Select(frame => Encoding.UTF8.GetString(frame.Payload)) + .ToList(); + Assert.That(delivered, Is.EqualTo(new[] { "REAL" })); + } + } + [Test] public void APermanentFailureInResendsEndsOnlyThatPeersSession() { From 25ea551bd5a2056c979b498415efab2119b4d057 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:54:33 +0200 Subject: [PATCH 20/33] cultnet-ts: an operation handler's result belongs to the generation its request arrived on When that generation ends while the handler runs (a goodbye, a refusal, a timeout, a new Connect from the same endpoint), the response is not sent and a failure sends no second goodbye: a client that connected again owns the endpoint. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/src/operation-service.ts | 7 ++ packages/cultnet-ts/test/cultnet.test.ts | 83 ++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index f74c579d..de41e405 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -226,6 +226,11 @@ async function handleServerDatagram( sessions.delete(key); return true; } + // A request belongs to the session generation it arrived on. If that generation ends while a + // handler runs (a goodbye, a refusal, a timeout, a new Connect from the same endpoint), what + // the handler returns or throws belongs to no live session, and nothing of it may reach the + // endpoint: a client that connected again there owns it now. + const generation = peer.session.generation; for (const frame of result.delivered) { if (frame.channelId !== "schema") continue; try { @@ -239,6 +244,7 @@ async function handleServerDatagram( peer.handling -= 1; peer.lastHandledAtMs = Date.now(); } + if (peer.session.generation !== generation) return true; const payload = encode(encodeCultNetMessageForWire(response, "cultnet.schema.v0")); for (const responsePacket of peer.session.sendMany("schema", payload, { reliable: true, @@ -247,6 +253,7 @@ async function handleServerDatagram( maxFragmentBytes: options.maxFragmentBytes ?? 2048, })) sendPacket(key, peer, responsePacket); } catch { + if (peer.session.generation !== generation) return true; // The session recorded this request's sequence, so keeping it would // acknowledge the retransmit of a request that was never handled. endSession(sessions, key, peer, sendPacket); diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index e6b62bdb..a6f19800 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -2825,6 +2825,89 @@ test("server-mode transport admits a restarted client and keeps its session when } }); +// A request belongs to the session generation it arrived on. A client that connects again +// from the same endpoint while a handler runs owns that endpoint now: nothing the handler +// returns or throws for the old generation may reach it. +async function restartDuringHandler(settle: (release: { resolve: () => void; reject: (error: Error) => void }) => void) { + let release: { resolve: () => void; reject: (error: Error) => void } | undefined; + const server = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + handler: async (request) => { + if (request.messageId === "held") { + await new Promise((resolve, reject) => { release = { resolve, reject: (error) => reject(error) }; }); + } + return { + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: "gamecult.eve.plugin_abi.response.v1", + payloadEncoding: "messagepack-base64", + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "sai-sidecar", + }; + }, + }); + const connectionId = 0x43554c54; + const port = Number(new URL(server.endpoint).port); + const socket = await bindUdpSocket(); + let session = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const answered: string[] = []; + const goodbyes: string[] = []; + socket.on("message", (wire) => { + const packet = decodeRudpPacket(wire); + if (packet.packetType === "disconnect") goodbyes.push(Buffer.from(packet.payload ?? []).toString("utf8")); + const result = session.receive(packet, Date.now()); + if (packet.packetType === "accept" || packet.reliable) { + socket.send(encodeRudpPacket(session.createAckForReceived(packet.sequence)), port, "127.0.0.1"); + } + for (const frame of result.delivered) { + answered.push((decodeMsgpack(frame.payload) as { messageId: string }).messageId); + } + }); + const toServer = (packet: CultNetRudpPacket) => socket.send(encodeRudpPacket(packet), port, "127.0.0.1"); + const request = (messageId: string) => Buffer.from(encode(encodeCultNetMessageForWire({ + schemaVersion: "cultnet.operation_request.v0", + messageId, + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + } satisfies CultNetOperationRequestMessage))); + try { + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the first Accept"); + toServer(session.send("schema", request("held"), { reliable: true, ordered: true })); + await waitFor(() => release !== undefined, "the held handler"); + + session = new CultNetRudpSession({ connectionId, initialSequence: 7 }); + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the restarted client's Accept"); + settle(release!); + await new Promise((resolve) => setTimeout(resolve, 100)); + toServer(session.send("schema", request("after-restart"), { reliable: true, ordered: true })); + await waitFor(() => answered.includes("after-restart"), "the restarted client's answer"); + + assert.deepEqual(answered, ["after-restart"], "the old generation's response reached the new session"); + assert.deepEqual(goodbyes, [], "a goodbye for the old generation reached the new session"); + assert.equal(session.connected, true); + } finally { + socket.close(); + await server.close(); + } +} + +test("an operation response for a generation a new Connect replaced is not sent into the new session", async () => { + await restartDuringHandler((release) => release.resolve()); +}); + +test("an operation handler that fails after a new Connect replaced its generation says no goodbye to the new session", async () => { + await restartDuringHandler((release) => release.reject(new Error("handler failed"))); +}); + test("RUDP operation service admits a restarted client on the same address", async () => { const server = await startCultNetOperationServer({ runtimeId: "sai-sidecar", From a288bb037aff0eaf90b758d34b28728b350e142f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 15:56:17 +0200 Subject: [PATCH 21/33] cultmesh-py: a peer's message that handling cannot serve ends that peer's session, never the RUDP thread The RUDP loop caught only ValueError, so a subscribe frame with `schemaIds: 5` raised TypeError in _handle_connection_message and ended the thread for every peer. Any exception from handling one peer's messages now ends that peer's session with a goodbye, as a refused packet does, and is logged by its type alone: the error's message can quote what the peer sent. Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-py/src/cultmesh_py/server.py | 52 ++++++++++++------- packages/cultmesh-py/tests/test_cultmesh.py | 31 +++++++++++ 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index be8cd903..413441c3 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import socket import threading import time @@ -32,6 +33,8 @@ from cultnet_py.cultmesh_contracts import ( CultMeshVerseCatalog, ) +_LOGGER = logging.getLogger(__name__) + @dataclass(frozen=True) class _DatabaseSubscription: @@ -278,24 +281,37 @@ class CultMeshLocalServer: if result.disconnected: peers.pop(remote_addr, None) continue - for delivered in result.delivered: - if delivered.channel_id != "schema": - continue - try: - message = msgpack.unpackb(delivered.payload, raw=False) - except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): - continue - if not isinstance(message, dict): - continue - responses = self._handle_connection_message(message, peer.subscriptions) - for response in responses: - if unsendable is None: - unsendable = self._send_rudp_schema_frame( - rudp_socket, - remote_addr, - peer.session, - msgpack.packb(response, use_bin_type=True), - ) + try: + for delivered in result.delivered: + if delivered.channel_id != "schema": + continue + try: + message = msgpack.unpackb(delivered.payload, raw=False) + except (msgpack.ExtraData, msgpack.FormatError, msgpack.StackError, ValueError): + continue + if not isinstance(message, dict): + continue + responses = self._handle_connection_message(message, peer.subscriptions) + for response in responses: + if unsendable is None: + unsendable = self._send_rudp_schema_frame( + rudp_socket, + remote_addr, + peer.session, + msgpack.packb(response, use_bin_type=True), + ) + except Exception as error: # noqa: BLE001 - one peer's message never ends the thread + # Whatever a peer's message makes the handling raise is that peer's failure. + # The session recorded the frame's sequence, so it cannot be kept: a retransmit + # would be acknowledged and never handled. The session ends, never the thread, + # and the peer is told. The log names the error type only, because the error's + # own message can quote what the peer sent. + _LOGGER.warning( + "CultMesh RUDP server could not handle a peer's message (%s); that peer's session ended.", + type(error).__name__, + ) + self._end_refused_peer(rudp_socket, peers, remote_addr) + continue if unsendable is None and ( packet.reliable or packet.packet_type == CultNetRudpPacketType.DATA or result.delivered ): diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 7c26949d..9b936ca1 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -4823,6 +4823,37 @@ class CultMeshRudpSendFailureTests(unittest.TestCase): finally: server.stop() + def test_a_message_whose_handling_fails_ends_only_that_peers_session(self) -> None: + import msgpack # type: ignore + + canary = "canary-7f3e1c" + # Well-formed frames whose fields have the wrong type: handling raises TypeError. + frames = [ + {"schemaVersion": "cultnet.database_subscribe.v0", "subscriptionId": canary, "schemaIds": 5}, + {"schemaVersion": "cultnet.database_subscribe.v0", "subscriptionId": canary, "recordKeys": True}, + ] + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + y, _ = self._client(server, "peer-y") + for frame in frames: + with self.subTest(frame=frame): + x, _ = self._client(server, "peer-x") + with self.assertLogs("cultmesh_py.server", level="WARNING") as logs: + x.send("schema", msgpack.packb(frame, use_bin_type=True)) + for _ in range(200): + x.receive_once() + if x.disconnect_reason is not None: + break + time.sleep(0.005) + self.assertEqual(x.disconnect_reason, b"session refused a packet") + self.assertTrue(server._rudp_thread.is_alive()) + self.assertIsNotNone(self._served(y), "Y is still served") + self.assertEqual(len(logs.output), 1) + self.assertIn("TypeError", logs.output[0]) + self.assertNotIn(canary, logs.output[0]) + finally: + server.stop() + def _raw_peer(self, server: Any) -> tuple[socket.socket, Any]: """A peer driven packet by packet, to deliver frames in an order a client would not.""" from cultnet_py import CultNetRudpSession, CultNetRudpSessionOptions, encode_rudp_packet From de8e2e1b88a59410ff415e88e5fcbc17cb523046 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:02:27 +0200 Subject: [PATCH 22/33] cultnet-ts: an operation server's send failure belongs to the generation that sent it A permanent failure reported after that generation ended (the client connected again from the same endpoint) ended the generation that replaced it and said goodbye to the new client. The callback now acts only while the generation that sent the datagram is live, which is the one in `sessions`. That makes two guards unreachable, so they are deleted: `peer.unsendable` before the goodbye (the first permanent failure ends the generation, so a second callback from it is already ignored) and the conditional delete (a live generation is the entry at its key). A refusal's goodbye is sent best-effort, as the transport and the document server send theirs, instead of through the failure handling meant for datagrams a session owes. Tests: late failure after a reconnect, for the transport's generation check and for the operation server. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/src/operation-service.ts | 25 +-- .../cultnet-ts/test/rudp-send-failure.test.ts | 142 ++++++++++++++++++ 2 files changed, 158 insertions(+), 9 deletions(-) diff --git a/packages/cultnet-ts/src/operation-service.ts b/packages/cultnet-ts/src/operation-service.ts index de41e405..90760591 100644 --- a/packages/cultnet-ts/src/operation-service.ts +++ b/packages/cultnet-ts/src/operation-service.ts @@ -81,16 +81,20 @@ export async function startCultNetOperationServer( let packetsDropped = 0; let sendFailures = 0; const sendPacket: SendPacket = (key, peer, packet) => { + const generation = peer.session.generation; sendRudpDatagram(socket, encodeRudpPacket(packet), peer.remote.port, peer.remote.address, (error) => { if (!isPermanentSendError(error)) { sendFailures += 1; return; } - if (peer.unsendable) return; + // A failure reported after the generation that sent the datagram ended is owed by + // nothing: that generation's end already told the peer, and a client that connected + // again from the endpoint owns it now. A live generation is the one in `sessions`. + if (peer.session.generation !== generation) return; // The datagram can never be sent as built: the session that owes it ends, never // the server, and the peer is told. peer.unsendable = true; - if (sessions.get(key) === peer) sessions.delete(key); + sessions.delete(key); const goodbye = encodeRudpPacket(peer.session.endUnsendableSession(error)); sendRudpDatagram(socket, goodbye, peer.remote.port, peer.remote.address, () => {}); }); @@ -99,7 +103,7 @@ export async function startCultNetOperationServer( // What a datagram carries is the sender's business: a rejection here must // drop and count the packet, never become an unhandled rejection that ends // the process. - handleServerDatagram(sessions, connectionId, options, wire, remote, sendPacket).then( + handleServerDatagram(sessions, connectionId, options, wire, remote, socket, sendPacket).then( admitted => { if (!admitted) packetsDropped += 1; }, () => { packetsDropped += 1; }, ); @@ -180,6 +184,7 @@ async function handleServerDatagram( options: CultNetOperationServerOptions, wire: Buffer, remote: RemoteInfo, + socket: Socket, sendPacket: SendPacket, ): Promise { let packet: CultNetRudpPacket; @@ -217,7 +222,7 @@ async function handleServerDatagram( try { result = peer.session.receive(packet, Date.now()); } catch { - endSession(sessions, key, peer, sendPacket); + endSession(sessions, key, peer, socket); return false; } if (result.reply) sendPacket(key, peer, result.reply); @@ -256,7 +261,7 @@ async function handleServerDatagram( if (peer.session.generation !== generation) return true; // The session recorded this request's sequence, so keeping it would // acknowledge the retransmit of a request that was never handled. - endSession(sessions, key, peer, sendPacket); + endSession(sessions, key, peer, socket); return false; } } @@ -269,17 +274,19 @@ async function handleServerDatagram( /** * Ends a session that took a packet it could not serve, and tells the peer. The * reset comes first, or the goodbye's ack field would acknowledge the very - * packet that was refused. A session already replaced under the same key is left. + * packet that was refused. The goodbye is best-effort: the session ends whether + * or not the peer hears it. */ function endSession( sessions: Map, key: string, peer: RemoteSession, - sendPacket: SendPacket, + socket: Socket, ): void { - if (sessions.get(key) === peer) sessions.delete(key); + sessions.delete(key); peer.session.resetPeerState(); - sendPacket(key, peer, peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + const goodbye = encodeRudpPacket(peer.session.createDisconnect(Buffer.from("session refused a packet", "utf8"))); + sendRudpDatagram(socket, goodbye, peer.remote.port, peer.remote.address, () => {}); } function parseRudpEndpoint(endpoint: string): { host: string; port: number } { diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts index edcf0d08..cb62d199 100644 --- a/packages/cultnet-ts/test/rudp-send-failure.test.ts +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -6,13 +6,17 @@ import assert from "node:assert/strict"; import dgram, { type RemoteInfo, type Socket } from "node:dgram"; import test from "node:test"; +import { encode } from "@msgpack/msgpack"; import { CultNetRudpSession, CultNetRudpSocketTransportConnection, + decodeRudpPacket, + encodeCultNetMessageForWire, encodeRudpPacket, invokeCultNetOperation, isPermanentSendError, startCultNetOperationServer, + type CultNetOperationRequestMessage, type CultNetOperationResponseMessage, } from "../src"; @@ -291,3 +295,141 @@ test("an operation call the network refuses rejects with a timeout and does not uncaught.stop(); } }); + +// A send's failure can be reported after the generation that sent it ended: the client +// connected again from the same endpoint in the meantime. That failure is owed by nothing; +// it may not end the generation that replaced it or say goodbye to the endpoint. + +/** Makes the next send to `port` fail as can never pass as built, reported `delayMs` later. */ +function failNextSendLate(socket: Socket, port: number, delayMs: number): { arm: () => void } { + let armed = false; + const send = socket.send.bind(socket) as (...args: unknown[]) => void; + Object.assign(socket, { + send: (message: Uint8Array, toPort: number, address: string, callback?: (error: Error | null) => void) => { + if (toPort !== port || !armed) { + send(message, toPort, address, callback); + return; + } + armed = false; + setTimeout(() => callback?.(Object.assign(new Error("late"), { code: "EMSGSIZE" })), delayMs); + }, + }); + return { arm: () => { armed = true; } }; +} + +/** A client driven packet by packet from one socket, so it can connect again from the same endpoint. */ +async function rawClient(serverPort: number, connectionId: number) { + const socket = await bindUdpSocket(); + const received: ReturnType[] = []; + socket.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const toServer = (packet: Parameters[0]) => socket.send(encodeRudpPacket(packet), serverPort, "127.0.0.1"); + const connect = async (initialSequence: number) => { + const session = new CultNetRudpSession({ connectionId, initialSequence }); + received.length = 0; + toServer(session.createConnect(Date.now())); + await waitFor(() => received.some((packet) => packet.packetType === "accept"), "the Accept"); + const accept = received.find((packet) => packet.packetType === "accept")!; + session.receive(accept, Date.now()); + toServer(session.createAckForReceived(accept.sequence)); + return session; + }; + return { socket, received, toServer, connect }; +} + +test("a transport ignores a send failure reported after the generation that sent it ended", async () => { + const serverSocket = await bindUdpSocket(); + const connectionId = 0x10203096; + const uncaught = watchUncaught(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 1_000, + }); + const reasons: string[] = []; + server.on("disconnect", ({ reason }: { reason: Uint8Array }) => reasons.push(Buffer.from(reason).toString("utf8"))); + const client = await rawClient(udpPort(serverSocket), connectionId); + const late = failNextSendLate(serverSocket, udpPort(client.socket), 150); + try { + await client.connect(10); + late.arm(); + server.send("schema", Buffer.from("old", "utf8")); + await client.connect(5_000); + await sleep(250); + + assert.deepEqual(reasons, []); + assert.equal(server.connected, true); + assert.deepEqual(client.received.filter((packet) => packet.packetType === "disconnect"), []); + assert.deepEqual(uncaught.errors, []); + } finally { + uncaught.stop(); + client.socket.close(); + server.close(); + } +}); + +test("an operation server ignores a send failure reported after the generation that sent it ended", async () => { + const connectionId = 0x43554c54; + const createSocket = dgram.createSocket; + let serverSocket: Socket | undefined; + Object.assign(dgram, { + createSocket: (...args: Parameters) => { + serverSocket = createSocket(...args); + return serverSocket; + }, + }); + const server = await startCultNetOperationServer({ + runtimeId: "operations", + handler: (request): CultNetOperationResponseMessage => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: request.payloadSchema, + payloadEncoding: request.payloadEncoding, + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "operations", + }), + }).finally(() => Object.assign(dgram, { createSocket })); + const client = await rawClient(Number(new URL(server.endpoint).port), connectionId); + const late = failNextSendLate(serverSocket!, udpPort(client.socket), 150); + const request = (session: CultNetRudpSession, messageId: string) => session.sendMany("schema", encode(encodeCultNetMessageForWire({ + schemaVersion: "cultnet.operation_request.v0", + messageId, + serviceId: "service", + operation: "operation", + payloadSchema: "payload", + payloadEncoding: "messagepack", + payload: new Uint8Array(), + diagnostics: [], + sourceRuntimeId: "caller", + } satisfies CultNetOperationRequestMessage, "cultnet.schema.v0")), { reliable: true, ordered: true, nowMs: Date.now() }); + try { + const old = await client.connect(10); + // The response to this request is the next send, and it fails after the client reconnected. + late.arm(); + for (const packet of request(old, "old")) client.toServer(packet); + const renewed = await client.connect(5_000); + await sleep(250); + assert.deepEqual(client.received.filter((packet) => packet.packetType === "disconnect"), []); + assert.equal(server.sendFailures, 0); + + // The session that replaced it is still served. + let read = client.received.length; + let answered = 0; + for (const packet of request(renewed, "after")) client.toServer(packet); + await waitFor(() => { + for (; read < client.received.length; read += 1) { + const packet = client.received[read]!; + if (packet.packetType === "data") answered += renewed.receive(packet, Date.now()).delivered.length; + } + return answered > 0; + }, "the renewed session's response"); + } finally { + client.socket.close(); + await server.close(); + } +}); From 2d76b8d1b6315ee85a4b6cb328b346183a57fe45 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:02:27 +0200 Subject: [PATCH 23/33] Pin the reset before a refusal's goodbye (cultmesh-ts, cultmesh-py) and C#'s once-guard The document servers' refusal tests sent an unreliable fragment, whose sequence the session never records, so dropping the reset before the goodbye survived. They now refuse a reliable fragment and check the goodbye does not acknowledge it. C#: a BeforeSend seam orders a caller's failing send between the polling thread's ack and its send, so both threads end the same session; the peer hears one goodbye. Co-Authored-By: Claude Opus 5.5 --- packages/cultmesh-py/tests/test_cultmesh.py | 31 ++++++++++++++++ .../rudp-document-server-send-failure.test.ts | 13 +++++-- src/GameCult.Networking/CultNetTransport.cs | 4 +++ .../CultNetRudpSendFailureTests.cs | 36 +++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-py/tests/test_cultmesh.py b/packages/cultmesh-py/tests/test_cultmesh.py index 9b936ca1..4c6afabc 100644 --- a/packages/cultmesh-py/tests/test_cultmesh.py +++ b/packages/cultmesh-py/tests/test_cultmesh.py @@ -4868,6 +4868,37 @@ class CultMeshRudpSendFailureTests(unittest.TestCase): session.receive(accept, 0) return sock, session + def test_the_goodbye_for_a_refused_reliable_frame_does_not_acknowledge_it(self) -> None: + import dataclasses + + from cultnet_py import CultNetRudpSendOptions, encode_rudp_packet + + server = CultMesh.serve_node(CultMesh.create_node(runtime_id="mesh-server")) + try: + sock, session = self._raw_peer(server) + # A reliable fragment whose metadata the session refuses: fragment id 0 names no set. + # The session records its sequence before it refuses it. + (fragment,) = session.send_many("schema", b"x", CultNetRudpSendOptions(reliable=True, ordered=True)) + refused = dataclasses.replace(fragment, fragment_id=0, fragment_index=0, fragment_count=2) + sock.sendto(encode_rudp_packet(refused), ("127.0.0.1", server.port)) + goodbye = None + for _ in range(40): + try: + wire, _ = sock.recvfrom(65535) + except TimeoutError: + continue + packet = decode_rudp_packet(wire) + if packet.packet_type == CultNetRudpPacketType.DISCONNECT: + goodbye = packet + break + self.assertIsNotNone(goodbye) + self.assertEqual(goodbye.payload, b"session refused a packet") + distance = goodbye.ack - refused.sequence + acknowledged = distance == 0 or (0 < distance <= 32 and goodbye.ack_mask & (1 << (distance - 1)) != 0) + self.assertFalse(acknowledged, "the goodbye acknowledged the refused frame") + finally: + server.stop() + def test_responses_after_one_that_can_never_be_sent_are_not_sent(self) -> None: from cultnet_py import CultNetRudpSendOptions, encode_rudp_packet diff --git a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts index 534a9440..64f19111 100644 --- a/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts +++ b/packages/cultmesh-ts/test/rudp-document-server-send-failure.test.ts @@ -189,8 +189,9 @@ test("a packet the session refuses ends only that peer's session, and the peer i session.receive(arrived.find((packet) => packet.packetType === "accept")!, Date.now()); y = await served.peer("y"); - // A fragment whose metadata the session refuses: fragment id 0 names no set. - const [fragment] = session.sendMany("schema", new Uint8Array([1]), { reliable: false }); + // A reliable fragment whose metadata the session refuses: fragment id 0 names no set. + // The session records its sequence before it refuses it. + const [fragment] = session.sendMany("schema", new Uint8Array([1]), { reliable: true, ordered: true }); raw.send( encodeRudpPacket({ ...fragment!, fragmentId: 0, fragmentIndex: 0, fragmentCount: 2 }), served.server.bind.port, @@ -199,6 +200,14 @@ test("a packet the session refuses ends only that peer's session, and the peer i await waitFor(() => arrived.some((packet) => packet.packetType === "disconnect"), "the goodbye"); const goodbye = arrived.find((packet) => packet.packetType === "disconnect")!; assert.equal(Buffer.from(goodbye.payload ?? []).toString("utf8"), "session refused a packet"); + // The goodbye's ack field does not acknowledge the refused frame, or the peer would + // count as delivered a frame that was never handled. + const acknowledged = + goodbye.ack === fragment!.sequence || + (goodbye.ack > fragment!.sequence && + goodbye.ack - fragment!.sequence <= 32 && + (goodbye.ackMask & (1 << (goodbye.ack - fragment!.sequence - 1))) !== 0); + assert.equal(acknowledged, false, "the goodbye acknowledged the refused frame"); assert.deepEqual(served.closed, [(raw.address() as { port: number }).port]); y.send(put("y-after")); diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index dcfe3b6a..7b09fe6a 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -3209,6 +3209,7 @@ namespace GameCult.Networking private SocketException? SendPacket(EndPoint remoteEndPoint, CultNetRudpPacket packet) { var wire = CultNetRudpPacketCodec.Encode(packet); + BeforeSend?.Invoke(remoteEndPoint, packet); try { if (UnsendableAfter.TryGetValue(remoteEndPoint, out var remaining)) @@ -3246,6 +3247,9 @@ namespace GameCult.Networking // Peers whose next datagram after this many fails permanently, once. internal Dictionary UnsendableAfter { get; } = new Dictionary(); + // Runs before each datagram is sent, to order another thread's work between two sends. + internal Action? BeforeSend { get; set; } + private static string RemoteKey(EndPoint endpoint) { return endpoint.ToString() ?? string.Empty; diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index 70f9acc7..0ae010dc 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -313,6 +313,42 @@ namespace GameCult.Networking.Tests } } + [Test] + public void ASessionEndedByACallerAndThePollingThreadTogetherSaysGoodbyeOnce() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var ended = RecordDisconnects(server); + // The polling thread acknowledges X's frame outside the session's gate. Just before + // that ack goes out, a caller's send to X fails after its first fragment and ends the + // session; then the ack can never be sent either. + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Ack) + return; + server.BeforeSend = null; + server.UnsendableAfter[x.EndPoint] = 1; + var caller = new Thread(() => + { + try { server.SendSchema(x.Server, new byte[3000]); } + catch (SocketException) { } + }); + caller.Start(); + caller.Join(); + server.UnsendableAfter[x.EndPoint] = 0; + }; + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(ReliableFrame(x, "from-x")), listenerEndPoint); + server.ReceiveOnce(); + server.PollResends(); + + var goodbyes = Drain(x.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Disconnect).ToList(); + Assert.That(goodbyes, Has.Count.EqualTo(1)); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(IsUnsendableReason(x.Server.DisconnectReason), Is.True); + } + } + [Test] public void APermanentFailureInResendsEndsOnlyThatPeersSession() { From 73fbb264d655405e0a4dab9e2bd2e79854f48615 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:02:28 +0200 Subject: [PATCH 24/33] cultnet-ts: a caller's send larger than a UDP datagram throws EMSGSIZE and takes nothing Node reports an oversized datagram only to the send's callback, so the caller's send used to succeed and the session ended later. The transport now checks the largest datagram of the send against the socket family's UDP limit (65,507 bytes for IPv4, 65,527 for IPv6) before the session builds anything: the send throws, nothing is queued, and no sequence is taken, as the Rust and C# listeners withdraw it. Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/src/rudp.ts | 16 +++++++ .../cultnet-ts/test/rudp-send-failure.test.ts | 43 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index da431cd9..e68293d3 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -30,6 +30,10 @@ import { const RUDP_MAGIC = [0x43, 0x4e, 0x52, 0x30] as const; // CNR0 const RUDP_VERSION = 0; const RUDP_FIXED_HEADER_BYTES = 36; +/// The largest UDP payload a datagram can carry: 65,535 less the UDP header and, +/// over IPv4, the minimal IP header. IPv6's header is outside its payload length. +const MAX_UDP4_DATAGRAM_BYTES = 65_507; +const MAX_UDP6_DATAGRAM_BYTES = 65_527; const MAX_CHANNEL_ID_BYTES = 255; export const CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS = 32; const RUDP_RECEIVED_SEQUENCE_WINDOW = 4_096; @@ -1037,6 +1041,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement readonly #mode: "client" | "server"; readonly #resendTimer: NodeJS.Timeout; readonly #maxFragmentBytes: number | undefined; + readonly #maxDatagramBytes: number; #remoteHost: string | undefined; #remotePort: number | undefined; #closed = false; @@ -1068,6 +1073,8 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement const address = this.#socket.address(); const localPort = typeof address === "string" ? undefined : address.port; const localHost = typeof address === "string" ? undefined : address.address; + this.#maxDatagramBytes = + typeof address !== "string" && String(address.family).endsWith("6") ? MAX_UDP6_DATAGRAM_BYTES : MAX_UDP4_DATAGRAM_BYTES; this.profile = createRudpTransportProfile(options.runtimeId, { transportId: options.transportId, host: localHost, @@ -1109,6 +1116,15 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } send(channelId: string, payload: Uint8Array): void { + // A datagram larger than UDP can carry can never be sent, and Node reports that + // only later, to the send's callback. Checked here it is the caller's error, as + // in the other runtimes: the send throws, and nothing of it is queued or takes a + // sequence. Every datagram of a send is at most one fragment of its payload. + const largestPayload = + this.#maxFragmentBytes === undefined ? payload.byteLength : Math.min(payload.byteLength, this.#maxFragmentBytes); + if (RUDP_FIXED_HEADER_BYTES + Buffer.byteLength(channelId, "utf8") + largestPayload > this.#maxDatagramBytes) { + throw Object.assign(new Error("RUDP datagram is larger than UDP can carry."), { code: "EMSGSIZE" }); + } const packets = this.#session.sendMany(channelId, payload, { ...channelOptions(channelId), nowMs: Date.now(), diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts index cb62d199..803b1e15 100644 --- a/packages/cultnet-ts/test/rudp-send-failure.test.ts +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -433,3 +433,46 @@ test("an operation server ignores a send failure reported after the generation t await server.close(); } }); + +test("a caller's send larger than a UDP datagram throws and takes nothing from the session", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203097; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + }); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + }); + const frames: number[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(frame.payload.byteLength)); + // 36 header bytes and the six of "schema": the largest payload one IPv4 datagram carries. + const largest = 65_507 - 36 - "schema".length; + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); + await waitFor(() => client.outstandingReliablePacketCount === 0, "the Connect acknowledged"); + + client.send("schema", new Uint8Array(largest)); + assert.throws(() => client.send("schema", new Uint8Array(largest + 1)), { code: "EMSGSIZE" }); + assert.equal(client.stats.framesSent, 1); + client.send("schema", Buffer.from("next", "utf8")); + + // The refused send took no sequence: the next frame is not held behind a gap. + await waitFor(() => frames.length === 2, "both sent frames"); + assert.deepEqual(frames, [largest, 4]); + assert.equal(client.connected, true); + await waitFor(() => client.outstandingReliablePacketCount === 0, "both frames acknowledged"); + } finally { + client.close(); + server.close(); + } +}); From 31b6c09d64b360c7d00996156508660e5edfed30 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:08:51 +0200 Subject: [PATCH 25/33] cultnet-ts: the send-failure tests build operation requests the contract accepts Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/test/rudp-send-failure.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts index 803b1e15..3078dbf7 100644 --- a/packages/cultnet-ts/test/rudp-send-failure.test.ts +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -281,8 +281,8 @@ test("an operation call the network refuses rejects with a timeout and does not serviceId: "service", operation: "operation", payloadSchema: "payload", - payloadEncoding: "messagepack", - payload: new Uint8Array(), + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", diagnostics: [], sourceRuntimeId: "caller", }, @@ -402,8 +402,8 @@ test("an operation server ignores a send failure reported after the generation t serviceId: "service", operation: "operation", payloadSchema: "payload", - payloadEncoding: "messagepack", - payload: new Uint8Array(), + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", diagnostics: [], sourceRuntimeId: "caller", } satisfies CultNetOperationRequestMessage, "cultnet.schema.v0")), { reliable: true, ordered: true, nowMs: Date.now() }); From d54110fe24c8d78b942bc9d39a5de74588b73c13 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 16:14:20 +0200 Subject: [PATCH 26/33] cultnet-ts: the send-failure tests send requests without response-only fields Co-Authored-By: Claude Opus 5.5 --- packages/cultnet-ts/test/rudp-send-failure.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts index 3078dbf7..a84c03d3 100644 --- a/packages/cultnet-ts/test/rudp-send-failure.test.ts +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -283,7 +283,6 @@ test("an operation call the network refuses rejects with a timeout and does not payloadSchema: "payload", payloadEncoding: "messagepack-base64", payload: "gaZzY2hlbWE=", - diagnostics: [], sourceRuntimeId: "caller", }, { runtimeId: "caller", timeoutMs: 400 }, @@ -404,7 +403,6 @@ test("an operation server ignores a send failure reported after the generation t payloadSchema: "payload", payloadEncoding: "messagepack-base64", payload: "gaZzY2hlbWE=", - diagnostics: [], sourceRuntimeId: "caller", } satisfies CultNetOperationRequestMessage, "cultnet.schema.v0")), { reliable: true, ordered: true, nowMs: Date.now() }); try { From 3b9ffac2fe90bb7d71cac2706258e5a701a3a2d1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:23:30 +0200 Subject: [PATCH 27/33] C#: a send failure's goodbye belongs to the generation that sent Send ended an unsendable session after releasing SessionGate, and the goodbye was built under the gate but sent outside it. If the polling thread admitted a new Connect from the same endpoint in that window, the ended generation's goodbye reached the new client: it saw Data, Accept, Disconnect and believed itself disconnected while the listener held it connected. Every path that finds a permanent send failure now reads the session's generation under the gate at the send, and EndUnsendableSession builds and sends the goodbye under the gate only while that generation is unchanged. The generation check replaces the UnsendableReason once-guard: a second failure in the same generation finds it already ended. The test orders the new Connect inside the send, before the fragment that can never be sent, and asserts the new client hears no Disconnect and stays connected on both sides. It fails on the pre-fix code (Data, Accept, Disconnect) and kills the mutant that drops the check. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 40 ++++++++++++------ .../CultNetRudpSendFailureTests.cs | 41 +++++++++++++++++++ 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 7b09fe6a..24839d41 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -2841,10 +2841,12 @@ namespace GameCult.Networking if (peer == null) throw new ArgumentNullException(nameof(peer)); SocketException? unsendable = null; var failedIndex = 0; + long generation; lock (peer.SessionGate) { var firstSequence = peer.Session.NextSequence; var packets = peer.Session.SendMany(channelId, payload, ChannelSendOptions(channelId), _maxFragmentBytes); + generation = peer.Session.Generation; unsendable = SendPackets(peer, packets, out failedIndex); // A caller-directed send that can never succeed is the caller's error and queues // nothing. Only when earlier fragments are already on the wire has the peer seen @@ -2856,7 +2858,7 @@ namespace GameCult.Networking { // The polling thread retires the peer; this thread only ends its session. if (failedIndex > 0) - EndUnsendableSession(peer, unsendable); + EndUnsendableSession(peer, unsendable, generation); throw unsendable; } _stats.FramesSent++; @@ -2973,8 +2975,10 @@ namespace GameCult.Networking { var repeats = false; SocketException? unsendableReply = null; + long admittedGeneration; lock (admitted.SessionGate) { + admittedGeneration = admitted.Session.Generation; if (!admitted.Session.ConnectRepeats(packet)) { admitted.Session.ResetPeerState(); @@ -2998,7 +3002,7 @@ namespace GameCult.Networking if (repeats) { if (unsendableReply != null) - EndUnsendablePeer(admitted, unsendableReply); + EndUnsendablePeer(admitted, unsendableReply, admittedGeneration); return true; } } @@ -3044,6 +3048,7 @@ namespace GameCult.Networking CultNetRudpReceiveResult? outcome; CultNetRudpPacket? acknowledgement = null; SocketException? unsendable = null; + long existingGeneration = 0; lock (existingPeer.SessionGate) { try @@ -3065,6 +3070,7 @@ namespace GameCult.Networking } else { + existingGeneration = existingPeer.Session.Generation; if (outcome.Reply != null) unsendable = SendPacket(existingPeer.RemoteEndPoint, outcome.Reply); unsendable ??= SendPackets(existingPeer, outcome.ReadyToSend, out _); @@ -3107,7 +3113,7 @@ namespace GameCult.Networking if (acknowledgement != null && unsendable == null) unsendable = SendPacket(existingPeer.RemoteEndPoint, acknowledgement); if (unsendable != null) - EndUnsendablePeer(existingPeer, unsendable); + EndUnsendablePeer(existingPeer, unsendable, existingGeneration); return true; } @@ -3123,10 +3129,15 @@ namespace GameCult.Networking foreach (var peer in _peers.Values.ToArray()) { SocketException? unsendable; + long generation; lock (peer.SessionGate) - unsendable = SendPackets(peer, peer.Session.DueResends(NowMs()), out _); + { + var due = peer.Session.DueResends(NowMs()); + generation = peer.Session.Generation; + unsendable = SendPackets(peer, due, out _); + } if (unsendable != null) - EndUnsendablePeer(peer, unsendable); + EndUnsendablePeer(peer, unsendable, generation); } } @@ -3152,25 +3163,28 @@ namespace GameCult.Networking // An admitted peer owes a packet that can never be sent as built. Inside a poll that ends the // peer's session, never the poll: the peer is told, and PeerDisconnected carries a reason that // names the error. - private void EndUnsendablePeer(CultNetRudpSocketServerPeer peer, SocketException error) + private void EndUnsendablePeer(CultNetRudpSocketServerPeer peer, SocketException error, long generation) { - EndUnsendableSession(peer, error); + EndUnsendableSession(peer, error, generation); RetireUnsendablePeers(); } // Ends the session and tells the peer, on whichever thread found the failure. The peer stays in - // _peers until the polling thread retires it. - private void EndUnsendableSession(CultNetRudpSocketServerPeer peer, SocketException error) + // _peers until the polling thread retires it. The failure belongs to the generation the thread + // read under the gate when it sent; if that generation has already ended (another thread + // ended it over its own failure, or a new Connect from the endpoint replaced it), there is + // nothing left to end, and a goodbye would reach the session that now owns the endpoint. The + // goodbye is built and sent under the gate, so no Connect is admitted between the two. + private void EndUnsendableSession(CultNetRudpSocketServerPeer peer, SocketException error, long generation) { - CultNetRudpPacket goodbye; lock (peer.SessionGate) { - if (peer.UnsendableReason != null) + if (peer.Session.Generation != generation) return; - goodbye = peer.Session.EndUnsendable(error); + var goodbye = peer.Session.EndUnsendable(error); peer.UnsendableReason = goodbye.Payload; + SendPacket(peer.RemoteEndPoint, goodbye); } - SendPacket(peer.RemoteEndPoint, goodbye); _unsendablePeers.Enqueue(peer); } diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index 0ae010dc..c9287c12 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -313,6 +313,47 @@ namespace GameCult.Networking.Tests } } + // A send that fails after its first fragment ends the generation it began in, and only that + // one. Here X connects again while the send is under way, and the polling thread admits the + // new Connect before the fragment that can never be sent. The hook runs that poll inside the + // send so the order is fixed; in production it runs on the polling thread between the send + // and the failure's handling. The goodbye belonged to the ended generation, and the client + // must not hear it: it would believe itself disconnected while the listener holds it + // connected. + [Test] + public void ASendFailureAfterANewConnectReplacedThePeerSaysNoGoodbyeToTheNewSession() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var stale = x.Server; + Drain(x.Socket); + x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint); + + var fragments = 0; + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Data || ++fragments < 2) + return; + server.BeforeSend = null; + server.ReceiveOnce(); + server.UnsendableAfter[x.EndPoint] = 0; + }; + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + server.ReceiveOnce(); + + var packets = Drain(x.Socket); + Assert.That(packets.Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect), + "the ended generation's goodbye reached the session that replaced it"); + x.Session.Receive(packets.Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + Assert.That(x.Session.Connected, Is.True); + var replacing = server.Peers.Single(p => p.RemoteEndPoint.Equals(x.EndPoint)); + Assert.That(replacing, Is.Not.SameAs(stale)); + Assert.That(replacing.Connected, Is.True); + } + } + [Test] public void ASessionEndedByACallerAndThePollingThreadTogetherSaysGoodbyeOnce() { From a44e189e0caa1d2788ae00b2c3bc194119902bbb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:23:40 +0200 Subject: [PATCH 28/33] C#: a peer's undecodable message ends that peer's session, never the poll RudpCultNetSchemaServer decoded each schema frame outside its per-message backstop, so a malformed payload (0xc1) or a well-formed message with an unknown schemaVersion threw MessagePackSerializationException out of PollAvailableAsync and PollOnceAsync. The drain aborted: a good peer's request in the same drain went unserved, a consumer loop that did not catch died, and the bad frame was already acknowledged while its peer stayed connected. Decoding now runs inside the backstop. A peer whose message cannot be decoded or handled has its session ended: the transport's new EndRefusedPeer sends the refusal goodbye under the gate, counts the drop in PacketsDropped, drops that peer's frames still queued, forgets the peer and raises PeerDisconnected. A session that already ended (replaced by a new Connect, or ended by a send failure) gets no goodbye, since the endpoint may belong to a new session. Other peers in the drain are still served. Ending the session matches cultmesh-py's server, where one peer's bad message ends that peer; runtime parity is the reason. The log names the fault's type only and never its message, because the decoder's message quotes the peer's input (the unknown schemaVersion). The dispatch-backstop test now asserts that rule instead of the handler message it used to require. Tests, each failing on the pre-fix code: a malformed payload and an unknown schemaVersion each leave the good peer served without Poll throwing (also through PollOnceAsync); the bad peer's session ends, counted and told; the log carries the type and not the canary value. Co-Authored-By: Claude Opus 5.5 --- .../CultNetRudpSchemaServer.cs | 37 ++-- src/GameCult.Networking/CultNetTransport.cs | 29 +++ .../CultNetRudpSchemaServerPeerInputTests.cs | 196 ++++++++++++++++++ .../CultNetSelectionFixBatchTests.cs | 7 +- 4 files changed, 248 insertions(+), 21 deletions(-) create mode 100644 tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs diff --git a/src/GameCult.Networking/CultNetRudpSchemaServer.cs b/src/GameCult.Networking/CultNetRudpSchemaServer.cs index e19fccc3..3112f07b 100644 --- a/src/GameCult.Networking/CultNetRudpSchemaServer.cs +++ b/src/GameCult.Networking/CultNetRudpSchemaServer.cs @@ -288,23 +288,22 @@ namespace GameCult.Networking if (!string.Equals(delivered.Frame.ChannelId, "schema", StringComparison.Ordinal)) return false; - var message = CultNetSchemaMessageSerialization.Deserialize(delivered.Frame.Payload); - if (!_handlers.TryGetValue(message.GetType(), out var handler) || handler == null) - { - return false; - } - - // R-AG: a handler is expected to catch its own typed refusals (HandleSnapshotRequestV1Async - // catches the three selection exceptions and answers cursor_invalid/selection_invalid/ - // reference_outside_target on the wire, R-AM) - this catch is the backstop for whatever a - // handler does not, so an untyped exception from a malformed/hostile message can never - // reach the poll loop and stall the server for every other peer. It still answers nothing - // on the wire (an exception this generic carries no typed refusal to report) and drops the - // one malformed dispatch, exactly as an unregistered handler already does just above - - // R-AM: but it always logs. A backstop that swallows silently is a compensator, and an - // untested, unlogged one hid the SM-6 ArgumentNullException regression for a whole batch. + // A peer's message that cannot be decoded (malformed bytes, an unknown schemaVersion) or + // handled is that peer's failure, never the poll's: the drain goes on serving every other + // peer. The transport already acknowledged the frame, so the peer's session cannot be kept + // (a retransmit would never be handled). It ends, the peer is told, and the drop is counted + // and logged, as cultmesh-py's server does. A handler is still expected to answer its own + // typed refusals on the wire (HandleSnapshotRequestV1Async does, R-AM); this is the backstop + // for whatever it does not. The log names the fault's type only: its message can quote + // what the peer sent. try { + var message = CultNetSchemaMessageSerialization.Deserialize(delivered.Frame.Payload); + if (!_handlers.TryGetValue(message.GetType(), out var handler) || handler == null) + { + return false; + } + var result = handler.DynamicInvoke(message, new RudpCultNetSchemaServerPeer(this, delivered.Peer)); if (result is Task task) { @@ -313,11 +312,11 @@ namespace GameCult.Networking } catch (Exception ex) { - // DynamicInvoke wraps a synchronously-thrown handler fault in TargetInvocationException, - // whose own .Message is the generic "Exception has been thrown by the target of an - // invocation." - log the real fault, not that wrapper. + // DynamicInvoke wraps a synchronously-thrown handler fault in TargetInvocationException; + // name the real fault, not that wrapper. var reported = ex is System.Reflection.TargetInvocationException { InnerException: { } inner } ? inner : ex; - Logger.LogError($"CultNet RUDP schema dispatch failed for {message.GetType().Name}: {reported.Message}"); + Logger.LogError($"CultNet RUDP schema server could not decode or handle a peer's message ({reported.GetType().Name}); that peer's session ended."); + _transport.EndRefusedPeer(delivered.Peer); return false; } return true; diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index 24839d41..ba63cf46 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -3188,6 +3188,35 @@ namespace GameCult.Networking _unsendablePeers.Enqueue(peer); } + // Polling thread only. Ends the session of a peer whose delivered frame the application could not + // decode or handle. The frame is already acknowledged, so the session cannot be kept: a + // retransmit would never be handled. The peer is told, the drop is counted, and the peer's + // frames still queued are not delivered. A session that already ended (a new Connect replaced + // it, or a send failure ended it) owes no goodbye; the endpoint may belong to a new session. + internal void EndRefusedPeer(CultNetRudpSocketServerPeer peer) + { + if (peer == null) throw new ArgumentNullException(nameof(peer)); + _stats.PacketsDropped++; + lock (peer.SessionGate) + { + if (peer.Session.Ended) + return; + SendPacket(peer.RemoteEndPoint, peer.Session.EndRefused()); + } + + var kept = _deliveredFrames.Where(frame => !ReferenceEquals(frame.Peer, peer)).ToArray(); + _deliveredFrames.Clear(); + foreach (var frame in kept) + _deliveredFrames.Enqueue(frame); + + var key = RemoteKey(peer.RemoteEndPoint); + if (!_peers.TryGetValue(key, out var current) || !ReferenceEquals(current, peer)) + return; + _peers.Remove(key); + peer.DisconnectReason = CultNetRudpSession.RefusedPacketReason; + PeerDisconnected?.Invoke(peer); + } + // Polling thread only. A peer a new Connect already replaced was reported then, and its key now // names the peer that replaced it. private void RetireUnsendablePeers() diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs new file mode 100644 index 00000000..74fc1d67 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs @@ -0,0 +1,196 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using GameCult.Logging; +using MessagePack; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// A schema server's poll serves every peer whatever one peer sends. A message that cannot be + /// decoded or handled is that peer's failure: its session ends, the drop is counted and logged, + /// and the drain goes on to the other peers. + /// + public class CultNetRudpSchemaServerPeerInputTests + { + private const uint ConnectionId = 0x10203096; + private const string Canary = "canary-5f1c9e0b"; + + private sealed class Client + { + public Socket Socket = null!; + public CultNetRudpSession Session = null!; + public EndPoint EndPoint => Socket.LocalEndPoint!; + } + + private sealed class CapturingLogger : ILogger + { + public List Errors { get; } = new(); + public void LogInfo(string message) { } + public void LogWarning(string message) { } + public void LogError(string message) => Errors.Add(message); + public void LogDebug(string message) { } + } + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + return socket; + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + private static async Task Connect(RudpCultNetSchemaServer server) + { + var client = new Client + { + Socket = Bind(), + Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }) + }; + client.Socket.SendTo(CultNetRudpPacketCodec.Encode(client.Session.CreateConnect(0)), server.LocalEndPoint); + await server.PollAvailableAsync(8); + client.Session.Receive(Drain(client.Socket).Single(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + return client; + } + + private static void SendSchema(RudpCultNetSchemaServer server, Client client, byte[] payload) => + client.Socket.SendTo( + CultNetRudpPacketCodec.Encode(client.Session.Send("schema", payload, new CultNetRudpSendOptions { Reliable = true, Ordered = true })), + server.LocalEndPoint); + + private static byte[] Malformed() => new byte[] { 0xc1 }; + + // Well-formed MessagePack naming a schema version no runtime knows. The decoder's own error + // quotes that version. + private static byte[] UnknownSchemaVersion() => + MessagePackSerializer.Serialize( + new Dictionary { ["schemaVersion"] = "cultnet." + Canary + ".v0", ["messageId"] = Canary }, + CultNetSchemaMessageSerialization.Options); + + private sealed class Scene : IDisposable + { + public RudpCultNetSchemaServer Server = null!; + public Client Bad = null!; + public Client Good = null!; + public CapturingLogger Logger = new(); + public List<(string MessageId, EndPoint From)> Served = new(); + public List Ended = new(); + + public void Dispose() + { + Server.Dispose(); + Bad.Socket.Dispose(); + Good.Socket.Dispose(); + } + } + + // The bad peer's frame is read first, so the drain reaches the good peer only by getting past it. + private static async Task BadThenGood(byte[] badPayload) + { + var scene = new Scene + { + Server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-peer-input", + ConnectionId = ConnectionId + }) + }; + scene.Server.Logger = scene.Logger; + scene.Server.OnCultNet((message, peer) => + scene.Served.Add((message.MessageId, peer.RemoteEndPoint))); + scene.Server.PeerDisconnected += peer => scene.Ended.Add(((RudpCultNetSchemaServerPeer)peer).RemoteEndPoint); + scene.Bad = await Connect(scene.Server); + scene.Good = await Connect(scene.Server); + + SendSchema(scene.Server, scene.Bad, badPayload); + SendSchema(scene.Server, scene.Good, CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "good-request" })); + return scene; + } + + [Test] + public async Task AMalformedPayloadDoesNotStopTheDrainServingAnotherPeer() + { + using var scene = await BadThenGood(Malformed()); + + Assert.DoesNotThrowAsync(async () => await scene.Server.PollAvailableAsync(16)); + + Assert.That(scene.Served, Is.EqualTo(new[] { ("good-request", scene.Good.EndPoint) })); + } + + [Test] + public async Task AnUnknownSchemaVersionDoesNotStopTheDrainServingAnotherPeer() + { + using var scene = await BadThenGood(UnknownSchemaVersion()); + + Assert.DoesNotThrowAsync(async () => await scene.Server.PollAvailableAsync(16)); + + Assert.That(scene.Served, Is.EqualTo(new[] { ("good-request", scene.Good.EndPoint) })); + } + + [Test] + public async Task PollOnceDoesNotThrowForAPeersUndecodableMessage() + { + using var scene = await BadThenGood(Malformed()); + + Assert.DoesNotThrowAsync(async () => + { + for (var attempt = 0; attempt < 16; attempt++) + await scene.Server.PollOnceAsync(); + }); + + Assert.That(scene.Served, Is.EqualTo(new[] { ("good-request", scene.Good.EndPoint) })); + } + + [Test] + public async Task APeerWhoseMessageCannotBeDecodedHasItsSessionEnded() + { + using var scene = await BadThenGood(Malformed()); + var dropped = scene.Server.Stats.PacketsDropped; + + try { await scene.Server.PollAvailableAsync(16); } + catch (MessagePackSerializationException) { Assert.Fail("the poll threw for a peer's input"); } + + // The frame was already acknowledged, so the session cannot be kept: the peer is told, + // the listener forgets it, and the drop is counted. + Assert.That(scene.Ended, Is.EqualTo(new[] { scene.Bad.EndPoint })); + Assert.That(scene.Server.Peers.Select(peer => peer.RemoteEndPoint), Is.EqualTo(new[] { scene.Good.EndPoint })); + Assert.That(scene.Server.Stats.PacketsDropped, Is.EqualTo(dropped + 1)); + var goodbye = Drain(scene.Bad.Socket).Single(packet => packet.PacketType == CultNetRudpPacketType.Disconnect); + Assert.That(scene.Bad.Session.Receive(goodbye, 1).Disconnected, Is.True); + Assert.That(Drain(scene.Good.Socket).Select(packet => packet.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(scene.Logger.Errors, Has.Count.EqualTo(1)); + } + + [Test] + public async Task TheDropIsLoggedByErrorTypeWithoutEchoingWhatThePeerSent() + { + using var scene = await BadThenGood(UnknownSchemaVersion()); + + try { await scene.Server.PollAvailableAsync(16); } + catch (MessagePackSerializationException) { Assert.Fail("the poll threw for a peer's input"); } + + // The decoder's own error message quotes the unknown version, which is the peer's input. + Assert.That(scene.Logger.Errors, Has.Count.EqualTo(1)); + Assert.That(scene.Logger.Errors[0], Does.Contain(nameof(MessagePackSerializationException))); + Assert.That(scene.Logger.Errors[0], Does.Not.Contain(Canary)); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs index 9b6959c0..e4c31d64 100644 --- a/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetSelectionFixBatchTests.cs @@ -796,8 +796,11 @@ namespace GameCult.Networking.Tests serverThread.Join(); } - Assert.That(logger.Errors, Has.Some.Contains("dispatch-backstop-probe"), - "the backstop must log the handler's own exception message, not swallow it silently"); + // The log names the fault's type. The fault's own message can quote what the peer sent, + // so it is never logged. + Assert.That(logger.Errors, Has.Some.Contains(nameof(InvalidOperationException)), + "the backstop must log the handler's fault, not swallow it silently"); + Assert.That(logger.Errors, Has.None.Contains("dispatch-backstop-probe")); } private sealed class CapturingLogger : ILogger From e9d5ae2cadfc6cb32b430748f1eca5f5452acdaf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 17:23:47 +0200 Subject: [PATCH 29/33] cultnet-ts: pin that a fragmented send over one datagram's size is sent The EMSGSIZE pre-check measures a send's largest datagram, which is one fragment when maxFragmentBytes is set. Nothing pinned that: a check that compared the whole payload (Soul's S-TM2) would refuse every large fragmented send, and large sends are live. A 70,000-byte send with 1,024-byte fragments must not throw and must arrive whole. The test kills the mutant whose size check ignores maxFragmentBytes. Co-Authored-By: Claude Opus 5.5 --- .../cultnet-ts/test/rudp-send-failure.test.ts | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts index a84c03d3..024f0817 100644 --- a/packages/cultnet-ts/test/rudp-send-failure.test.ts +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -474,3 +474,42 @@ test("a caller's send larger than a UDP datagram throws and takes nothing from t server.close(); } }); + +test("a fragmented send larger than one UDP datagram is sent, because each datagram carries one fragment", async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203095; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + }); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + maxFragmentBytes: 1024, + }); + const frames: number[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(frame.payload.byteLength)); + // Larger than any one datagram can carry, over IPv4 or IPv6; every fragment is 1,024 bytes. + const payloadBytes = 70_000; + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); + + assert.doesNotThrow(() => client.send("schema", new Uint8Array(payloadBytes))); + assert.equal(client.stats.framesSent, 1); + + await waitFor(() => frames.length === 1, "the reassembled frame"); + assert.deepEqual(frames, [payloadBytes]); + assert.equal(client.connected, true); + } finally { + client.close(); + server.close(); + } +}); From b37f898465e77040aa3ba418c8c261b7ee6d0a7d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:19:19 +0200 Subject: [PATCH 30/33] C#: a send failure on an already-ended session says no second goodbye 3b9ffac2 claimed that the generation check subsumed the UnsendableReason once-guard. It did not. A thread that reads the generation only after the session has ended carries the ended session's own generation, and the check lets it through. For example, the polling thread answers a Ping on a session that a caller's send has just ended but that has not yet been retired. When that Pong fails permanently, the thread says a second goodbye and overwrites the first failure's reason (Soul pass 4, PR4). EndUnsendableSession now also returns when the session has already ended. Two tests: - The polling thread answers a Ping on a session already ended, and the Pong can never be sent. Exactly one goodbye goes out, and the first failure's reason stands. The hook admits the Ping after the session ended and before the peer is queued for retirement. The test kills the mutant that drops the Ended check. - A failed send's goodbye is sent under the gate. A polling thread admitting a new Connect from the endpoint waits on the gate, so the old generation's goodbye never follows the new session's Accept. The test kills the mutant that sends the goodbye outside the gate. Co-Authored-By: Claude Opus 5.5 --- src/GameCult.Networking/CultNetTransport.cs | 8 +- .../CultNetRudpSendFailureTests.cs | 76 +++++++++++++++++++ 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index ba63cf46..dd8f3992 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -3172,14 +3172,16 @@ namespace GameCult.Networking // Ends the session and tells the peer, on whichever thread found the failure. The peer stays in // _peers until the polling thread retires it. The failure belongs to the generation the thread // read under the gate when it sent; if that generation has already ended (another thread - // ended it over its own failure, or a new Connect from the endpoint replaced it), there is - // nothing left to end, and a goodbye would reach the session that now owns the endpoint. The + // ended it over its own failure, or a new Connect from the endpoint replaced it), or the + // thread read it only after the session had ended (a Pong or ack answered on an ended + // session the polling thread has not yet retired), there is nothing left to end, and a + // goodbye would reach the session that now owns the endpoint or repeat the first one. The // goodbye is built and sent under the gate, so no Connect is admitted between the two. private void EndUnsendableSession(CultNetRudpSocketServerPeer peer, SocketException error, long generation) { lock (peer.SessionGate) { - if (peer.Session.Generation != generation) + if (peer.Session.Generation != generation || peer.Session.Ended) return; var goodbye = peer.Session.EndUnsendable(error); peer.UnsendableReason = goodbye.Payload; diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index c9287c12..7b327c02 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -354,6 +354,82 @@ namespace GameCult.Networking.Tests } } + // A caller's send ends X's session over a fragment it could never send. Before the listener + // retires X, the polling thread answers X's Ping on the ended session, and that Pong can never + // be sent either. The polling thread read the generation only after the session ended, so the + // generation alone cannot tell it the session is gone. It says no second goodbye, and the + // first failure's reason stands. The hook admits the Ping while the first goodbye is on its way, + // after the session ended and before the peer is queued for retirement. + [Test] + public void AFailureAnsweringAPingOnAnAlreadyEndedSessionSaysNoSecondGoodbye() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var ended = RecordDisconnects(server); + byte[]? firstReason = null; + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Disconnect) + return; + server.BeforeSend = null; + firstReason = x.Server.UnsendableReason; + server.UnsendableAfter[x.EndPoint] = 0; + server.ReceiveOnce(); + }; + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreatePing(Encoding.UTF8.GetBytes("ping"))), listenerEndPoint); + server.UnsendableAfter[x.EndPoint] = 1; + + Assert.Throws(() => server.SendSchema(x.Server, new byte[3000])); + server.ReceiveOnce(); + + var goodbyes = Drain(x.Socket).Where(p => p.PacketType == CultNetRudpPacketType.Disconnect).ToList(); + Assert.That(goodbyes, Has.Count.EqualTo(1)); + Assert.That(firstReason, Is.Not.Null); + Assert.That(x.Server.UnsendableReason, Is.SameAs(firstReason), "the second failure overwrote the first's reason"); + Assert.That(ended, Is.EqualTo(new[] { x.Server })); + Assert.That(x.Server.DisconnectReason, Is.SameAs(firstReason)); + } + } + + // The goodbye for a failed send is sent under the session's gate. The polling thread, admitting + // a new Connect from X, waits on that gate, so the ended generation's goodbye can never follow + // the new session's Accept. The hook starts the polling thread just before the goodbye leaves + // and gives it a second to get ahead if nothing holds it back. + [Test] + public void AFailedSendsGoodbyeLeavesBeforeANewConnectFromTheEndpointIsAdmitted() + { + var (server, listenerEndPoint, x, _) = TwoPeers(maxFragmentBytes: 1000); + using (server) + { + var stale = x.Server; + Drain(x.Socket); + x.Session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + x.Socket.SendTo(CultNetRudpPacketCodec.Encode(x.Session.CreateConnect(0)), listenerEndPoint); + + Thread? poller = null; + server.BeforeSend = (remote, packet) => + { + if (!remote.Equals(x.EndPoint) || packet.PacketType != CultNetRudpPacketType.Disconnect) + return; + server.BeforeSend = null; + poller = new Thread(() => server.ReceiveOnce()); + poller.Start(); + poller.Join(1000); + }; + server.UnsendableAfter[x.EndPoint] = 1; + Assert.Throws(() => server.SendSchema(stale, new byte[3000])); + Assert.That(poller, Is.Not.Null); + poller!.Join(); + + var types = Drain(x.Socket).Select(p => p.PacketType).ToList(); + Assert.That(types, Does.Contain(CultNetRudpPacketType.Disconnect)); + Assert.That(types, Does.Contain(CultNetRudpPacketType.Accept)); + Assert.That(types.IndexOf(CultNetRudpPacketType.Disconnect), Is.LessThan(types.IndexOf(CultNetRudpPacketType.Accept)), + "the ended generation's goodbye followed the new session's Accept"); + } + } + [Test] public void ASessionEndedByACallerAndThePollingThreadTogetherSaysGoodbyeOnce() { From ce63ff88dcf02e4e4932ae55998025104ea72e3d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:19:29 +0200 Subject: [PATCH 31/33] C#: a handler's fault ends the session without blaming the peer The schema server's backstop ended every session with "session refused a packet", including when the fault was the server's own: for example a handler's response too large to send, or a refusal the handler did not type. Decoding and handling are now two catches: - A message that cannot be decoded is the peer's. The session ends with the refusal reason. - A handler fault ends the session with "server could not handle a message". The transport's EndRefusedPeer becomes EndPeerSession(peer, reason). It reports whether it ended the session, so the log says when a response that failed after its first fragment had already ended it over the send. Before, the log reported that it ended a session that was already gone. The comment said that ending the session matched cultmesh-py. It does not: Python silently skips a malformed frame. The honest anchor is the C# TCP schema server, whose ServePeerAsync ends the one peer whose message fails to decode or handle. The comment now says so. Pinned, each by a test that kills its mutant: - a handler fault's goodbye carries the server's reason, not the refusal (mutant: the refusal reason); - the post-fragment-0 path logs that the session had already ended (mutant: a constant outcome); - EndPeerSession says no goodbye to a replaced peer's endpoint (mutant: the Ended check dropped); - a refused peer's frames still queued are never handled (mutant: the queue purge dropped). Co-Authored-By: Claude Opus 5.5 --- .../CultNetRudpSchemaServer.cs | 59 ++++++++--- src/GameCult.Networking/CultNetTransport.cs | 32 +++--- .../CultNetRudpSchemaServerPeerInputTests.cs | 97 +++++++++++++++++++ .../CultNetRudpSendFailureTests.cs | 22 +++++ 4 files changed, 184 insertions(+), 26 deletions(-) diff --git a/src/GameCult.Networking/CultNetRudpSchemaServer.cs b/src/GameCult.Networking/CultNetRudpSchemaServer.cs index 3112f07b..de8d0154 100644 --- a/src/GameCult.Networking/CultNetRudpSchemaServer.cs +++ b/src/GameCult.Networking/CultNetRudpSchemaServer.cs @@ -3,6 +3,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Net; using System.Net.Sockets; +using System.Text; using System.Threading.Tasks; using GameCult.Logging; @@ -169,6 +170,9 @@ namespace GameCult.Networking /// public CultNetTransportStats Stats => _transport.Stats; + // The listener this server drives, for tests that inject send failures through its seams. + internal CultNetRudpSocketTransportServer Transport => _transport; + /// /// Gets the currently tracked RUDP peers. /// @@ -283,27 +287,49 @@ namespace GameCult.Networking return new RudpCultNetSchemaPollResult(consumed, dispatched); } + /// + /// The goodbye's reason when a handler fails on a peer's message: the server could not handle + /// it, which says nothing about whether the peer's packet was well formed. + /// + internal static readonly byte[] HandlerFailedReason = Encoding.UTF8.GetBytes("server could not handle a message"); + + private static string SessionOutcome(bool ended) => + ended ? "that peer's session ended." : "that peer's session had already ended."; + private async Task DispatchAsync(CultNetRudpSocketServerFrame delivered) { if (!string.Equals(delivered.Frame.ChannelId, "schema", StringComparison.Ordinal)) return false; // A peer's message that cannot be decoded (malformed bytes, an unknown schemaVersion) or - // handled is that peer's failure, never the poll's: the drain goes on serving every other - // peer. The transport already acknowledged the frame, so the peer's session cannot be kept - // (a retransmit would never be handled). It ends, the peer is told, and the drop is counted - // and logged, as cultmesh-py's server does. A handler is still expected to answer its own - // typed refusals on the wire (HandleSnapshotRequestV1Async does, R-AM); this is the backstop - // for whatever it does not. The log names the fault's type only: its message can quote - // what the peer sent. + // handled ends that peer's session, never the poll: the drain goes on serving every other + // peer. The transport already acknowledged the frame, so the session cannot be kept (a + // retransmit would never be handled). The peer is told, and the drop is counted and logged. + // The C# TCP schema server (CultNetTcpSchemaTransport.ServePeerAsync) likewise ends the one + // peer whose message fails to decode or handle. A handler is still expected to answer its + // own typed refusals on the wire (HandleSnapshotRequestV1Async does, R-AM); this is the + // backstop for whatever it does not. The log names the fault's type only: its message can + // quote what the peer sent. + ICultNetSchemaMessage message; try { - var message = CultNetSchemaMessageSerialization.Deserialize(delivered.Frame.Payload); - if (!_handlers.TryGetValue(message.GetType(), out var handler) || handler == null) - { - return false; - } + message = CultNetSchemaMessageSerialization.Deserialize(delivered.Frame.Payload); + } + catch (Exception ex) + { + // The peer sent something no runtime can read: the refusal is the peer's. + var ended = _transport.EndPeerSession(delivered.Peer, CultNetRudpSession.RefusedPacketReason); + Logger.LogError($"CultNet RUDP schema server could not decode a peer's message ({ex.GetType().Name}); {SessionOutcome(ended)}"); + return false; + } + if (!_handlers.TryGetValue(message.GetType(), out var handler) || handler == null) + { + return false; + } + + try + { var result = handler.DynamicInvoke(message, new RudpCultNetSchemaServerPeer(this, delivered.Peer)); if (result is Task task) { @@ -313,10 +339,13 @@ namespace GameCult.Networking catch (Exception ex) { // DynamicInvoke wraps a synchronously-thrown handler fault in TargetInvocationException; - // name the real fault, not that wrapper. + // name the real fault, not that wrapper. The fault is the server's (a response too large + // to send, a handler bug) or a refusal the handler did not type, so the goodbye does not + // say the peer's packet was refused. A response that failed after its first fragment + // already ended the session over the send, and the log says so. var reported = ex is System.Reflection.TargetInvocationException { InnerException: { } inner } ? inner : ex; - Logger.LogError($"CultNet RUDP schema server could not decode or handle a peer's message ({reported.GetType().Name}); that peer's session ended."); - _transport.EndRefusedPeer(delivered.Peer); + var ended = _transport.EndPeerSession(delivered.Peer, HandlerFailedReason); + Logger.LogError($"CultNet RUDP schema server's {message.GetType().Name} handler failed ({reported.GetType().Name}); {SessionOutcome(ended)}"); return false; } return true; diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index dd8f3992..a6102de3 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -1462,10 +1462,17 @@ namespace GameCult.Networking /// sequence, so the peer's sequences are forgotten before the goodbye is built: its ack field /// would otherwise acknowledge the very frame the session refused. /// - internal CultNetRudpPacket EndRefused() + internal CultNetRudpPacket EndRefused() => EndWith(RefusedPacketReason); + + /// + /// Ends a session whose peer's frame the application above it refused or failed on, after the + /// session acknowledged it. As with , the peer's sequences are forgotten + /// before the goodbye is built. + /// + internal CultNetRudpPacket EndWith(byte[] reason) { ResetPeerState(); - return CreateDisconnect(RefusedPacketReason); + return CreateDisconnect(reason); } /// @@ -3191,19 +3198,21 @@ namespace GameCult.Networking } // Polling thread only. Ends the session of a peer whose delivered frame the application could not - // decode or handle. The frame is already acknowledged, so the session cannot be kept: a - // retransmit would never be handled. The peer is told, the drop is counted, and the peer's - // frames still queued are not delivered. A session that already ended (a new Connect replaced - // it, or a send failure ended it) owes no goodbye; the endpoint may belong to a new session. - internal void EndRefusedPeer(CultNetRudpSocketServerPeer peer) + // decode or handle, with a goodbye carrying the application's reason. The frame is already + // acknowledged, so the session cannot be kept: a retransmit would never be handled. The peer + // is told, the drop is counted, and the peer's frames still queued are not delivered. A + // session that already ended (a new Connect replaced it, or a send failure ended it) owes no + // goodbye, since the endpoint may belong to a new session; the call then returns false. + internal bool EndPeerSession(CultNetRudpSocketServerPeer peer, byte[] reason) { if (peer == null) throw new ArgumentNullException(nameof(peer)); + if (reason == null) throw new ArgumentNullException(nameof(reason)); _stats.PacketsDropped++; lock (peer.SessionGate) { if (peer.Session.Ended) - return; - SendPacket(peer.RemoteEndPoint, peer.Session.EndRefused()); + return false; + SendPacket(peer.RemoteEndPoint, peer.Session.EndWith(reason)); } var kept = _deliveredFrames.Where(frame => !ReferenceEquals(frame.Peer, peer)).ToArray(); @@ -3213,10 +3222,11 @@ namespace GameCult.Networking var key = RemoteKey(peer.RemoteEndPoint); if (!_peers.TryGetValue(key, out var current) || !ReferenceEquals(current, peer)) - return; + return true; _peers.Remove(key); - peer.DisconnectReason = CultNetRudpSession.RefusedPacketReason; + peer.DisconnectReason = reason; PeerDisconnected?.Invoke(peer); + return true; } // Polling thread only. A peer a new Connect already replaced was reported then, and its key now diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs index 74fc1d67..b8de674f 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSchemaServerPeerInputTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; using System.Net.Sockets; +using System.Text; using System.Threading.Tasks; using GameCult.Logging; using MessagePack; @@ -179,6 +180,102 @@ namespace GameCult.Networking.Tests Assert.That(scene.Logger.Errors, Has.Count.EqualTo(1)); } + // The peer's malformed message and a well-formed one after it arrive in one datagram's + // delivery: the second is held for order, and both are released together. The session ends + // on the first, so the second belongs to no session and is never handled. + [Test] + public async Task ARefusedPeersFramesStillQueuedAreNotHandled() + { + var logger = new CapturingLogger(); + using var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-peer-input", + ConnectionId = ConnectionId + }); + server.Logger = logger; + var served = new List(); + server.OnCultNet((message, _) => served.Add(message.MessageId)); + var bad = await Connect(server); + var options = new CultNetRudpSendOptions { Reliable = true, Ordered = true }; + var malformed = bad.Session.Send("schema", Malformed(), options); + var after = bad.Session.Send("schema", CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "after-the-refused-one" }), options); + bad.Socket.SendTo(CultNetRudpPacketCodec.Encode(after), server.LocalEndPoint); + bad.Socket.SendTo(CultNetRudpPacketCodec.Encode(malformed), server.LocalEndPoint); + + await server.PollAvailableAsync(16); + + Assert.That(served, Is.Empty); + Assert.That(server.Peers, Is.Empty); + Assert.That(logger.Errors, Has.Count.EqualTo(1)); + bad.Socket.Dispose(); + } + + private static async Task<(RudpCultNetSchemaServer Server, Client Client, CapturingLogger Logger)> ServerWhoseHandlerAnswers( + byte[] response, int? maxFragmentBytes, int? unsendableAfter = null) + { + var logger = new CapturingLogger(); + var server = new RudpCultNetSchemaServer(new RudpCultNetSchemaServerOptions + { + RuntimeId = "csharp-peer-input", + ConnectionId = ConnectionId, + MaxFragmentBytes = maxFragmentBytes + }); + server.Logger = logger; + server.OnCultNet((_, peer) => + { + if (unsendableAfter is { } count) + server.Transport.UnsendableAfter[peer.RemoteEndPoint] = count; + server.Transport.SendSchema(peer.TransportPeer, response); + }); + var client = await Connect(server); + SendSchema(server, client, CultNetSchemaMessageSerialization.Serialize( + new CultNetShardCatalogRequestMessage { MessageId = "ask" })); + return (server, client, logger); + } + + // A handler whose own response can never be sent (larger than a datagram, unfragmented) fails + // on the server's account, not the peer's. The session still ends, but its goodbye does not say + // the peer's packet was refused. + [Test] + public async Task AHandlerFaultEndsTheSessionWithAReasonThatDoesNotBlameThePeer() + { + var (server, client, logger) = await ServerWhoseHandlerAnswers(new byte[70_000], maxFragmentBytes: null); + using (server) + { + await server.PollAvailableAsync(16); + + var goodbye = Drain(client.Socket).Single(packet => packet.PacketType == CultNetRudpPacketType.Disconnect); + Assert.That(Encoding.UTF8.GetString(goodbye.Payload!), Is.EqualTo("server could not handle a message")); + Assert.That(server.Peers, Is.Empty); + Assert.That(logger.Errors, Has.Count.EqualTo(1)); + Assert.That(logger.Errors[0], Does.Contain("handler failed (SocketException)")); + Assert.That(logger.Errors[0], Does.Contain("that peer's session ended.")); + } + client.Socket.Dispose(); + } + + // A response that fails after its first fragment has already ended the session over the send, + // with a goodbye that names the send failure. The backstop ends nothing more and says so. + [Test] + public async Task AHandlerWhoseResponseEndedTheSessionIsLoggedAsAlreadyEnded() + { + var (server, client, logger) = await ServerWhoseHandlerAnswers(new byte[3000], maxFragmentBytes: 1000, unsendableAfter: 1); + using (server) + { + await server.PollAvailableAsync(16); + await server.PollAvailableAsync(16); + + var goodbyes = Drain(client.Socket).Where(packet => packet.PacketType == CultNetRudpPacketType.Disconnect).ToList(); + Assert.That(goodbyes, Has.Count.EqualTo(1)); + Assert.That(Encoding.UTF8.GetString(goodbyes[0].Payload!), Does.StartWith("packet could not be sent: ")); + Assert.That(server.Peers, Is.Empty); + Assert.That(logger.Errors, Has.Count.EqualTo(1)); + Assert.That(logger.Errors[0], Does.Contain("that peer's session had already ended.")); + } + client.Socket.Dispose(); + } + [Test] public async Task TheDropIsLoggedByErrorTypeWithoutEchoingWhatThePeerSent() { diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs index 7b327c02..81d255f7 100644 --- a/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs +++ b/tests/GameCult.Networking.Tests/CultNetRudpSendFailureTests.cs @@ -430,6 +430,28 @@ namespace GameCult.Networking.Tests } } + // An application ending a peer's session (a message it could not decode or handle) says no + // goodbye to a session that already ended: a new Connect replaced it, and the endpoint now + // belongs to the session that replaced it. + [Test] + public void EndingAReplacedPeersSessionSaysNoGoodbyeToTheSessionThatReplacedIt() + { + var (server, listenerEndPoint, x, _) = TwoPeers(); + using (server) + { + var (stale, replacing) = Reconnect(server, listenerEndPoint, x); + var ended = RecordDisconnects(server); + + Assert.That(server.EndPeerSession(stale, Encoding.UTF8.GetBytes("application reason")), Is.False); + + Assert.That(Drain(x.Socket).Select(p => p.PacketType), Has.None.EqualTo(CultNetRudpPacketType.Disconnect)); + Assert.That(x.Session.Connected, Is.True); + Assert.That(replacing.Connected, Is.True); + Assert.That(server.Peers, Has.Member(replacing)); + Assert.That(ended, Is.Empty); + } + } + [Test] public void ASessionEndedByACallerAndThePollingThreadTogetherSaysGoodbyeOnce() { From 1f04d5eda37574d39888d8d041b42d5a02fa51b4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:19:36 +0200 Subject: [PATCH 32/33] cultnet-ts: pin the one-fragment size check with a large fragment too With only 1 KiB fragments, a pre-check measuring a small multiple of a fragment (Soul's min(payload, frag*2)) still passed a 70,000-byte send. The fixture now also runs with 40,000-byte fragments. Any such multiple of that is over the datagram limit, so the mutant dies. Co-Authored-By: Claude Opus 5.5 --- .../cultnet-ts/test/rudp-send-failure.test.ts | 77 ++++++++++--------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/packages/cultnet-ts/test/rudp-send-failure.test.ts b/packages/cultnet-ts/test/rudp-send-failure.test.ts index 024f0817..4370d406 100644 --- a/packages/cultnet-ts/test/rudp-send-failure.test.ts +++ b/packages/cultnet-ts/test/rudp-send-failure.test.ts @@ -475,41 +475,46 @@ test("a caller's send larger than a UDP datagram throws and takes nothing from t } }); -test("a fragmented send larger than one UDP datagram is sent, because each datagram carries one fragment", async () => { - const serverSocket = await bindUdpSocket(); - const clientSocket = await bindUdpSocket(); - const connectionId = 0x10203095; - const server = new CultNetRudpSocketTransportConnection({ - runtimeId: "rudp-server", - socket: serverSocket, - mode: "server", - connectionId, - }); - const client = new CultNetRudpSocketTransportConnection({ - runtimeId: "rudp-client", - socket: clientSocket, - mode: "client", - remoteHost: "127.0.0.1", - remotePort: udpPort(serverSocket), - connectionId, - maxFragmentBytes: 1024, - }); - const frames: number[] = []; - server.on("frame", (frame: { payload: Uint8Array }) => frames.push(frame.payload.byteLength)); - // Larger than any one datagram can carry, over IPv4 or IPv6; every fragment is 1,024 bytes. - const payloadBytes = 70_000; - try { - client.connect(); - await waitFor(() => client.connected && server.connected, "the handshake"); +// The pre-check measures one fragment, the largest datagram a fragmented send makes. Fragments +// of 1 KiB and of 40,000 bytes both leave a 70,000-byte send legal; the second size is large +// enough that any small multiple of it would not be. +for (const maxFragmentBytes of [1024, 40_000]) { + test(`a fragmented send larger than one UDP datagram is sent, because each datagram carries one fragment (${maxFragmentBytes}-byte fragments)`, async () => { + const serverSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x10203095; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + }); + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(serverSocket), + connectionId, + maxFragmentBytes, + }); + const frames: number[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(frame.payload.byteLength)); + // Larger than any one datagram can carry, over IPv4 or IPv6. + const payloadBytes = 70_000; + try { + client.connect(); + await waitFor(() => client.connected && server.connected, "the handshake"); - assert.doesNotThrow(() => client.send("schema", new Uint8Array(payloadBytes))); - assert.equal(client.stats.framesSent, 1); + assert.doesNotThrow(() => client.send("schema", new Uint8Array(payloadBytes))); + assert.equal(client.stats.framesSent, 1); - await waitFor(() => frames.length === 1, "the reassembled frame"); - assert.deepEqual(frames, [payloadBytes]); - assert.equal(client.connected, true); - } finally { - client.close(); - server.close(); - } -}); + await waitFor(() => frames.length === 1, "the reassembled frame"); + assert.deepEqual(frames, [payloadBytes]); + assert.equal(client.connected, true); + } finally { + client.close(); + server.close(); + } + }); +} From 108071e8b3c031ee51c027d6c5323d2a0cd88502 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 20:19:44 +0200 Subject: [PATCH 33/33] Interop harness: ask cargo where the Rust peer is, and name missing prerequisites The harness assumed that the Rust interop peer builds under packages/cultnet-rs/target. When CARGO_TARGET_DIR or a cargo config's build.target-dir is set, cargo builds elsewhere, and every Rust lane failed with a spawn ENOENT. The path now comes from `cargo metadata`'s target_directory, read once on first use. The TypeScript peer imports @gamecult/cultcache-ts, and the Python peers import msgpack. When either was missing, the lanes died with import errors in a child's stderr or with timeouts. The first use of each peer now checks its prerequisite and fails with a message that names it and the command that provides it. Co-Authored-By: Claude Opus 5.5 --- .../test/interop/cultnet-interop.test.ts | 136 ++++++++++++------ 1 file changed, 95 insertions(+), 41 deletions(-) diff --git a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts index 756ef97d..598d6fe6 100644 --- a/packages/cultnet-ts/test/interop/cultnet-interop.test.ts +++ b/packages/cultnet-ts/test/interop/cultnet-interop.test.ts @@ -61,13 +61,6 @@ const cultMeshTsRoot = resolve(cultLibRoot, "packages", "cultmesh-ts"); const cultMeshQuicPeerScript = resolve(cultMeshTsRoot, "dist-test", "test", "interop", "cultmesh-quic-peer.js"); const quicNativeTestsProject = resolve(cultLibRoot, "tests", "GameCult.Mesh.Quic.Native.Tests", "GameCult.Mesh.Quic.Native.Tests.csproj"); const quicManagedTestsProject = resolve(cultLibRoot, "tests", "GameCult.Mesh.Quic.Tests", "GameCult.Mesh.Quic.Tests.csproj"); -const rustBinaryPath = resolve( - cultnetRsRoot, - "target", - "debug", - "examples", - process.platform === "win32" ? "cultnet_interop_peer.exe" : "cultnet_interop_peer", -); const discoveryGroup = "239.77.44.11"; const serveReadyTimeoutMs = 90_000; @@ -78,6 +71,67 @@ let csharpInteropPeerBuild: Promise | undefined; let kotlinInteropPeerBuild: Promise | undefined; let cultMeshTsInteropPeerBuild: Promise | undefined; +let rustInteropPeerPathCache: string | undefined; +let tsInteropPeerChecked = false; +let pythonInteropChecked = false; + +// Cargo decides where it builds: CARGO_TARGET_DIR, a config's build.target-dir, or the crate's own +// target directory. The peer's path is asked of cargo, not assumed. +function rustInteropPeerPath(): string { + if (rustInteropPeerPathCache !== undefined) { + return rustInteropPeerPathCache; + } + const metadata = spawnSync(cargoCommand, ["metadata", "--format-version", "1", "--no-deps"], { + cwd: cultnetRsRoot, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + if (metadata.status !== 0) { + throw new Error( + `The Rust interop peer's target directory is unknown: \`cargo metadata\` failed in ${cultnetRsRoot}.\n${metadata.stderr ?? metadata.error?.message ?? ""}`, + ); + } + const targetDirectory = (JSON.parse(metadata.stdout) as { target_directory: string }).target_directory; + rustInteropPeerPathCache = resolve( + targetDirectory, + "debug", + "examples", + process.platform === "win32" ? "cultnet_interop_peer.exe" : "cultnet_interop_peer", + ); + return rustInteropPeerPathCache; +} + +// The TypeScript peer imports @gamecult/cultcache-ts from its built output. Unbuilt, the peer dies +// on import and a lane fails with a spawn or timeout error that names neither. +function tsInteropPeerScript(): string { + if (!tsInteropPeerChecked) { + try { + require.resolve("@gamecult/cultcache-ts"); + } catch { + throw new Error( + "@gamecult/cultcache-ts is not built, and the TypeScript interop peer imports it. Run `npm run build --workspace packages/cultcache-ts` first.", + ); + } + tsInteropPeerChecked = true; + } + return tsPeerScript; +} + +// The Python peers import msgpack. Missing, each Python lane fails with a traceback in a child's +// stderr instead of a reason. +function pythonInteropCommand(): string { + if (!pythonInteropChecked) { + const probe = spawnSync(pythonCommand, ["-c", "import msgpack"], { encoding: "utf8" }); + if (probe.status !== 0) { + throw new Error( + `${pythonCommand} cannot import msgpack, which the Python interop peers need. Install the Python packages first: \`python -m pip install -e packages/cultcache-py -e packages/cultnet-py -e packages/cultmesh-py\`.\n${probe.stderr ?? probe.error?.message ?? ""}`, + ); + } + pythonInteropChecked = true; + } + return pythonCommand; +} + function resolvePythonCommand(): string { const codexPythonCommand = join(homedir(), ".cache", "codex-runtimes", "codex-primary-runtime", "dependencies", "python", "python.exe"); const candidates = process.platform === "win32" @@ -743,7 +797,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "start rust peer"); servers.push(await spawnServeProcess("rust", { - command: rustBinaryPath, + command: rustInteropPeerPath(), args: [ "serve", "--runtime-id", "rust-peer", @@ -765,7 +819,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state servers.push(await spawnServeProcess("ts", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-peer", "--runtime-kind", "node", @@ -805,7 +859,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "start python peer"); servers.push(await spawnServeProcess("python", { - command: pythonCommand, + command: pythonInteropCommand(), args: [ "-m", "cultnet_py.interop_peer", "serve", @@ -827,7 +881,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript dials rust"); const tsDial = await runJsonCommand("ts-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-client", "--runtime-kind", "node", @@ -846,7 +900,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript dials typescript over rudp"); const tsRudpDial = await runJsonCommand("ts-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-rudp-client", "--runtime-kind", "node", @@ -864,7 +918,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(tsRudpDial.fireReceipt.accepted, true); logInteropPhase("schema-v0", "rust dials csharp"); - const rustDial = await runJsonCommand("rust-dial", rustBinaryPath, [ + const rustDial = await runJsonCommand("rust-dial", rustInteropPeerPath(), [ "dial", "--runtime-id", "rust-client", "--runtime-kind", "rust", @@ -899,7 +953,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(csharpDial.fireReceipt.accepted, true); logInteropPhase("schema-v0", "python dials rust"); - const pythonDial = await runJsonCommand("python-dial", pythonCommand, [ + const pythonDial = await runJsonCommand("python-dial", pythonInteropCommand(), [ "-m", "cultnet_py.interop_peer", "dial", "--runtime-id", "python-client", @@ -917,7 +971,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state assert.equal(pythonDial.fireReceipt.accepted, true); logInteropPhase("schema-v0", "python dials python over rudp"); - const pythonRudpDial = await runJsonCommand("python-rudp-dial", pythonCommand, [ + const pythonRudpDial = await runJsonCommand("python-rudp-dial", pythonInteropCommand(), [ "-m", "cultnet_py.interop_peer", "dial", "--runtime-id", "python-rudp-client", @@ -937,7 +991,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript dials python"); const tsDialPython = await runJsonCommand("ts-dial-python", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-python-client", "--runtime-kind", "node", @@ -1257,7 +1311,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state logInteropPhase("schema-v0", "typescript discovery probe"); await expectProbePeers("ts-probe", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "probe", "--runtime-id", "ts-prober", "--discovery-port", String(discoveryPort), @@ -1266,7 +1320,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state ], cultNetTsRoot, {}, expectedPeers); logInteropPhase("schema-v0", "rust discovery probe"); - await expectProbePeers("rust-probe", rustBinaryPath, [ + await expectProbePeers("rust-probe", rustInteropPeerPath(), [ "probe", "--runtime-id", "rust-prober", "--discovery-port", String(discoveryPort), @@ -1285,7 +1339,7 @@ test("CultNet TS/Rust/C#/Python peers discover each other and exchange raw state ], cultLibRoot, {}, expectedPeers); logInteropPhase("schema-v0", "python discovery probe"); - await expectProbePeers("python-probe", pythonCommand, [ + await expectProbePeers("python-probe", pythonInteropCommand(), [ "-m", "cultnet_py.interop_peer", "probe", "--runtime-id", "python-prober", @@ -1319,7 +1373,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD }); servers.push(await spawnServeProcess("rust-rudp", { - command: rustBinaryPath, + command: rustInteropPeerPath(), args: [ "serve", "--runtime-id", "rust-rudp-peer", @@ -1341,7 +1395,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD servers.push(await spawnServeProcess("ts-rust-rudp", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-rust-rudp-peer", "--runtime-kind", "node", @@ -1361,7 +1415,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD logInteropPhase("ts-rust-full-rudp", "typescript dials rust over rudp"); const tsRudpDial = await runJsonCommand("ts-rust-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-rust-rudp-client", "--runtime-kind", "node", @@ -1379,7 +1433,7 @@ test("CultNet TypeScript and Rust full interop peers exchange schema-v0 over RUD assert.equal(tsRudpDial.fireReceipt.accepted, true); logInteropPhase("ts-rust-full-rudp", "rust dials typescript over rudp"); - const rustRudpDial = await runJsonCommand("rust-rudp-dial", rustBinaryPath, [ + const rustRudpDial = await runJsonCommand("rust-rudp-dial", rustInteropPeerPath(), [ "dial", "--runtime-id", "rust-rudp-client", "--runtime-kind", "rust", @@ -2186,7 +2240,7 @@ test("CultNet TypeScript and C# full interop peers exchange schema-v0 over RUDP" servers.push(await spawnServeProcess("ts-csharp-rudp", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-csharp-rudp-peer", "--runtime-kind", "node", @@ -2206,7 +2260,7 @@ test("CultNet TypeScript and C# full interop peers exchange schema-v0 over RUDP" logInteropPhase("ts-csharp-full-rudp", "typescript dials csharp over rudp"); const tsRudpDial = await runJsonCommand("ts-csharp-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-csharp-rudp-client", "--runtime-kind", "node", @@ -2289,7 +2343,7 @@ test("CultNet TypeScript and Kotlin full interop peers exchange schema-v0 over R servers.push(await spawnServeProcess("ts-kotlin-rudp", { command: process.execPath, args: [ - tsPeerScript, + tsInteropPeerScript(), "serve", "--runtime-id", "ts-kotlin-rudp-peer", "--runtime-kind", "node", @@ -2309,7 +2363,7 @@ test("CultNet TypeScript and Kotlin full interop peers exchange schema-v0 over R logInteropPhase("ts-kotlin-full-rudp", "typescript dials kotlin over rudp"); const tsRudpDial = await runJsonCommand("ts-kotlin-rudp-dial", process.execPath, [ - tsPeerScript, + tsInteropPeerScript(), "dial", "--runtime-id", "ts-kotlin-rudp-client", "--runtime-kind", "node", @@ -2962,7 +3016,7 @@ interface RunningPythonRudpPeer { } function spawnPythonRudpPeer(options: RudpServerSpawnOptions = {}): RunningPythonRudpPeer { - const child = spawn(pythonCommand, ["-c", pythonRudpPeerScript], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpPeerScript], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3018,7 +3072,7 @@ function spawnPythonRudpPeer(options: RudpServerSpawnOptions = {}): RunningPytho } function spawnPythonRudpMessagePeer(): RunningPythonRudpPeer { - const child = spawn(pythonCommand, ["-c", pythonRudpMessagePeerScript], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpMessagePeerScript], { cwd: cultcachePyRoot, env: { ...process.env, PYTHONPATH: cultcachePySrc }, stdio: ["ignore", "pipe", "pipe"], @@ -3066,7 +3120,7 @@ function spawnPythonRudpMessagePeer(): RunningPythonRudpPeer { } function spawnRustRudpServer(options: RudpServerSpawnOptions = {}): RunningPythonRudpPeer { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-serve-once", "--bind-host", "127.0.0.1", ...(options.clientPayload ? ["--client-payload", options.clientPayload] : []), @@ -3124,7 +3178,7 @@ function spawnRustRudpServer(options: RudpServerSpawnOptions = {}): RunningPytho } function spawnRustRudpMessageServer(): RunningPythonRudpPeer { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-serve-message-once", "--bind-host", "127.0.0.1", ], { @@ -3405,7 +3459,7 @@ function spawnKotlinRudpMessageServer(): RunningPythonRudpPeer { } function spawnPythonRudpClient(remotePort: number): Omit { - const child = spawn(pythonCommand, ["-c", pythonRudpClientScript, String(remotePort)], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpClientScript, String(remotePort)], { cwd: cultcachePyRoot, env: { ...process.env, PYTHONPATH: cultcachePySrc }, stdio: ["ignore", "pipe", "pipe"], @@ -3420,7 +3474,7 @@ function spawnPythonRudpClient(remotePort: number): Omit { - const child = spawn(pythonCommand, ["-c", pythonRudpMessageClientScript, String(remotePort)], { + const child = spawn(pythonInteropCommand(), ["-c", pythonRudpMessageClientScript, String(remotePort)], { cwd: cultcachePyRoot, env: { ...process.env, PYTHONPATH: cultcachePySrc }, stdio: ["ignore", "pipe", "pipe"], @@ -3517,7 +3571,7 @@ function spawnCSharpRudpMessageClient(remotePort: number): Omit { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-dial-once", "--target-host", "127.0.0.1", "--target-port", String(remotePort), @@ -3536,7 +3590,7 @@ function spawnRustRudpClient(remotePort: number): Omit { - const child = spawn(rustBinaryPath, [ + const child = spawn(rustInteropPeerPath(), [ "rudp-dial-message-once", "--target-host", "127.0.0.1", "--target-port", String(remotePort), @@ -3832,7 +3886,7 @@ async function runPythonCultMeshClient(port: number): Promise { " 'peers': [{'peerId': peer.peer_id, 'roles': list(peer.roles)} for peer in peers],", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3857,7 +3911,7 @@ async function runPythonCultNetRawClient(port: number): Promise { " 'shards': [{'shardId': shard.get('shardId'), 'epoch': shard.get('epoch')} for shard in shards.get('shards', [])],", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3894,7 +3948,7 @@ async function runPythonCultMeshNodeSync(port: number): Promise { " 'note': note,", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -3959,7 +4013,7 @@ async function runPythonCultMeshNodeEmit(port: number): Promise { " 'note': note,", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -4013,7 +4067,7 @@ async function runPythonCultNetSubscription(port: number): Promise { " 'recordKey': change.get('document', {}).get('recordKey'),", "}))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env, @@ -4067,7 +4121,7 @@ async function runPythonCultMeshPredictionReconcile(port: number): Promise " 'note': note,", " }))", ].join("\n"); - const { stdout } = await execFileAsync(pythonCommand, ["-c", script], { + const { stdout } = await execFileAsync(pythonInteropCommand(), ["-c", script], { cwd: cultcachePyRoot, env: { ...process.env,