diff --git a/packages/cultmesh-py/src/cultmesh_py/facade.py b/packages/cultmesh-py/src/cultmesh_py/facade.py index 53328aab..626b1dc7 100644 --- a/packages/cultmesh-py/src/cultmesh_py/facade.py +++ b/packages/cultmesh-py/src/cultmesh_py/facade.py @@ -157,7 +157,7 @@ class CultMesh: bind_host: str = "127.0.0.1", bind_port: int = 0, socket: socket_module.socket | None = None, - initial_sequence: int = 1, + initial_sequence: int | None = None, resend_delay_ms: int = 250, transport_id: str = "rudp", max_payload_bytes: int | None = None, @@ -191,7 +191,7 @@ class CultMesh: bind_host: str = "127.0.0.1", bind_port: int = 0, socket: socket_module.socket | None = None, - initial_sequence: int = 1, + initial_sequence: int | None = None, resend_delay_ms: int = 250, transport_id: str = "rudp", max_payload_bytes: int | None = None, diff --git a/packages/cultmesh-py/src/cultmesh_py/server.py b/packages/cultmesh-py/src/cultmesh_py/server.py index e0c7fdd0..fe4edd52 100644 --- a/packages/cultmesh-py/src/cultmesh_py/server.py +++ b/packages/cultmesh-py/src/cultmesh_py/server.py @@ -224,15 +224,19 @@ class CultMeshLocalServer: now_ms = _now_ms() peer = peers.get(remote_addr) if packet.packet_type == CultNetRudpPacketType.CONNECT: - peer = _RudpPeerConnection( - CultNetRudpSession( - CultNetRudpSessionOptions( - connection_id=self.rudp_connection_id, - resend_delay_ms=self.rudp_resend_delay_ms, + # The session decides whether this Connect repeats the one it + # accepted. Any other Connect is a new client session, with new + # subscriptions. + if peer is None or not peer.session.connect_repeats(packet): + peer = _RudpPeerConnection( + CultNetRudpSession( + CultNetRudpSessionOptions( + connection_id=self.rudp_connection_id, + resend_delay_ms=self.rudp_resend_delay_ms, + ) ) ) - ) - peers[remote_addr] = peer + peers[remote_addr] = peer self._send_rudp_packet( rudp_socket, remote_addr, diff --git a/packages/cultmesh-py/tests/test_rudp_server_generation.py b/packages/cultmesh-py/tests/test_rudp_server_generation.py new file mode 100644 index 00000000..967a03c2 --- /dev/null +++ b/packages/cultmesh-py/tests/test_rudp_server_generation.py @@ -0,0 +1,159 @@ +"""The CultMesh local server's RUDP loop asks each peer's session whether a Connect +repeats. A retransmitted Connect keeps the peer; a restarted client on the same +address is a new one, with none of the old one's subscriptions.""" +from __future__ import annotations + +import socket +import time +import unittest +from dataclasses import dataclass + +import msgpack +from cultcache_py import define_database_entry_type +from cultmesh_py import create_node +from cultmesh_py.server import CultMeshLocalServer +from cultnet_py import ( + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, + document_put_raw, +) +from cultnet_py.transport import CultNetRudpPacketType, decode_rudp_packet, encode_rudp_packet + +CONNECTION_ID = 0x10203091 +SCHEMA_ID = "mesh.rudp_generation_note.v1" +RELIABLE = CultNetRudpSendOptions(reliable=True, ordered=True) + + +@dataclass +class Note: + body: str + + +NOTE = define_database_entry_type( + "mesh.rudp_generation_note", + [("body", 0)], + cls=Note, + schema_id=SCHEMA_ID, + schema_name="mesh.rudp_generation_note", + schema_version=SCHEMA_ID, +) + + +def wire(message: dict) -> bytes: + return msgpack.packb(message, use_bin_type=True) + + +class RudpServerGenerationTests(unittest.TestCase): + def setUp(self) -> None: + node = create_node(runtime_id="rudp-generation") + node.database.register_document(NOTE) + self.server = CultMeshLocalServer(node=node, rudp_connection_id=CONNECTION_ID) + self.server.start() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.bind(("127.0.0.1", 0)) + self.sock.settimeout(0.02) + self.target = ("127.0.0.1", self.server.port) + # Every schema data packet the server sent, by sequence, whether or not the receiving + # session delivered it: a new server-side session numbers its packets afresh. + self.sent: dict[int, str] = {} + + def tearDown(self) -> None: + self.sock.close() + self.server.stop() + + def to_server(self, packet) -> None: + self.sock.sendto(encode_rudp_packet(packet), self.target) + + def pump(self, session: CultNetRudpSession, until, seconds: float = 2.0) -> bool: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if until(): + return True + try: + data, _ = self.sock.recvfrom(65535) + except TimeoutError: + continue + packet = decode_rudp_packet(data) + session.receive(packet, int(time.monotonic() * 1000)) + if packet.reliable or packet.packet_type == CultNetRudpPacketType.ACCEPT: + self.to_server(session.create_ack_for_received(packet.sequence)) + if packet.packet_type == CultNetRudpPacketType.DATA and packet.channel_id == "schema": + self.sent[packet.sequence] = msgpack.unpackb(packet.payload, raw=False).get("schemaVersion") + return until() + + def count(self, schema_version: str) -> int: + return sum(1 for value in self.sent.values() if value == schema_version) + + def new_session(self, initial_sequence: int) -> CultNetRudpSession: + return CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=initial_sequence)) + + def test_a_retransmitted_connect_keeps_the_peer_and_a_restarted_client_is_a_new_one(self) -> None: + first = self.new_session(50_000) + connect = first.create_connect(0, b"same") + self.to_server(connect) + self.assertTrue(self.pump(first, lambda: first.connected), "the first Accept") + hello = first.send( + "schema", wire({"schemaVersion": "cultnet.hello.v0", "runtimeId": "probe", "runtimeKind": "test"}), RELIABLE + ) + self.to_server(hello) + self.assertTrue(self.pump(first, lambda: self.count("cultnet.hello.v0") == 1), "the first answer") + self.to_server( + first.send( + "schema", + wire({ + "schemaVersion": "cultnet.database_subscribe.v0", + "messageId": "subscribe", + "subscriptionId": "old-incarnation", + "schemaIds": [SCHEMA_ID], + "includeSnapshot": False, + }), + RELIABLE, + ) + ) + self.pump(first, lambda: False, 0.2) + first_put = document_put_raw( + message_id="first-put", + key="note:a", + schema_id=SCHEMA_ID, + stored_at="2026-09-30T00:00:00Z", + payload=NOTE.encode_payload(Note("first")), + ) + self.to_server(first.send("schema", first_put.to_bytes(), RELIABLE)) + self.assertTrue( + self.pump(first, lambda: self.count("cultnet.database_change_raw.v0") == 1), + "the subscription did not notify its own connection", + ) + + # A retransmitted Connect repeats: the data already delivered is not delivered, or answered, again. + self.to_server(connect) + self.to_server(hello) + self.pump(first, lambda: False, 0.4) + self.assertEqual(self.count("cultnet.hello.v0"), 1, "the retransmitted Connect reset the peer") + + # The same client restarted on the same address, connection id and payload: a new peer. + second = self.new_session(7) + self.to_server(second.create_connect(0, b"same")) + self.assertTrue(self.pump(second, lambda: second.connected), "the restarted client's Accept") + self.to_server( + second.send( + "schema", wire({"schemaVersion": "cultnet.hello.v0", "runtimeId": "probe", "runtimeKind": "test"}), RELIABLE + ) + ) + self.assertTrue(self.pump(second, lambda: self.count("cultnet.hello.v0") == 2), "the restarted client's answer") + + # It inherited none of the old incarnation's subscriptions. + put = document_put_raw( + message_id="second-put", + key="note:a", + schema_id=SCHEMA_ID, + stored_at="2026-09-30T00:00:01Z", + payload=NOTE.encode_payload(Note("second")), + ) + self.to_server(second.send("schema", put.to_bytes(), RELIABLE)) + self.pump(second, lambda: False, 0.4) + self.assertEqual(self.count("cultnet.database_change_raw.v0"), 1, "the new peer inherited a subscription") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultmesh-rs/src/lib.rs b/packages/cultmesh-rs/src/lib.rs index d850f3b9..9c3e4d90 100644 --- a/packages/cultmesh-rs/src/lib.rs +++ b/packages/cultmesh-rs/src/lib.rs @@ -961,7 +961,7 @@ fn connect_rudp_client( mode: cultnet_rs::CultNetRudpSocketMode::Client, remote_addr: Some(target), connection_id: fresh_rudp_connection_epoch(), - initial_sequence: 1, + initial_sequence: None, resend_delay_ms, transport_id: Some(transport_id.to_string()), max_payload_bytes: None, @@ -1406,7 +1406,7 @@ mod tests { let mut session = cultnet_rs::CultNetRudpSession::new(cultnet_rs::CultNetRudpSessionOptions { connection_id: connect.connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 5, max_pending_reliable_packets: Some(64), }); diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index b99c7145..47348bfe 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -410,12 +410,20 @@ where packet: &CultNetRudpPacket, now: u64, ) -> Result { + // The session decides whether a Connect repeats the one it accepted. Any + // other Connect from this key is a new client session: the old one ends + // with its budget, and the new one starts fresh. + if self + .sessions + .get(&key) + .is_some_and(|entry| !entry.session.connect_repeats(packet)) + { + self.sessions.remove(&key); + } let reply = match self.sessions.get_mut(&key) { - // The same key is a retransmitted Connect for the existing epoch. - // A genuinely fresh client incarnation must choose a fresh id. Some(entry) => { entry.last_activity_monotonic_millis = now; - match entry.session.answer_repeated_connect(packet, now) { + match entry.session.accept_connect(packet, now, Vec::new()) { Ok(reply) => reply, Err(_) => { self.end_refused_session(key)?; @@ -429,7 +437,7 @@ where } let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: key.connection_id, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: duration_millis(self.options.resend_delay), max_pending_reliable_packets: Some( self.options.max_pending_reliable_packets_per_session, diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index fa741827..e43e4e59 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -160,7 +160,15 @@ fn server( } fn client(target: SocketAddr, id: u32) -> Result { - let socket = UdpSocket::bind("127.0.0.1:0")?; + client_on(UdpSocket::bind("127.0.0.1:0")?, target, id, 1) +} + +fn client_on( + socket: UdpSocket, + target: SocketAddr, + id: u32, + initial_sequence: u32, +) -> Result { socket.set_nonblocking(true)?; CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions { media_delivery: None, @@ -169,7 +177,7 @@ fn client(target: SocketAddr, id: u32) -> Result Result<()> } #[test] -fn duplicate_connect_preserves_epoch_and_fresh_epoch_is_separate() -> Result<()> { +fn a_second_connect_from_a_client_replaces_its_session_and_a_fresh_epoch_is_separate() -> Result<()> { let clock = Clock::new(45_000); let sink = Sink::default(); let mut server = server(Default::default(), clock, sink.clone(), Source::default())?; @@ -325,6 +333,97 @@ fn duplicate_connect_preserves_epoch_and_fresh_epoch_is_separate() -> Result<()> Ok(()) } +/// A client that restarts on the same address and connection id starts a new +/// session: its Connect carries a sequence the accepted one did not, so the +/// server does not mistake it for a retransmit, and what it sends is delivered. +#[test] +fn a_restarted_client_on_the_same_address_and_connection_id_is_admitted() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(46_000), sink.clone(), Source::default())?; + let target = server.local_addr()?; + let shared = UdpSocket::bind("127.0.0.1:0")?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, target, 93, 50_000)?; + connect(&mut server, &mut [&mut first])?; + for index in 0..3 { + send( + &mut first, + &CultNetMessage::DocumentPutRaw { + message_id: format!("before-{index}"), + document: document("before", vec![index]), + }, + )?; + } + for _ in 0..40 { + server.poll_once()?; + first.receive_once()?; + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 3); + + let mut restarted = client_on(twin, target, 93, 7)?; + connect(&mut server, &mut [&mut restarted])?; + assert_eq!(server.session_count(), 1); + send( + &mut restarted, + &CultNetMessage::DocumentPutRaw { + message_id: "after-restart".into(), + document: document("after", vec![9]), + }, + )?; + for _ in 0..40 { + server.poll_once()?; + restarted.receive_once()?; + } + let receipts = sink.0.lock().unwrap(); + assert!( + receipts.receipts.iter().any(|r| r.message_id == "after-restart"), + "the restarted client's frame was not delivered" + ); + Ok(()) +} + +/// A restarted client is a new session, and starts with a new payload budget: +/// what the old one spent is not charged to it. +#[test] +fn a_restarted_client_starts_with_a_fresh_payload_budget() -> Result<()> { + let put = |id: &str| CultNetMessage::DocumentPutRaw { + message_id: id.into(), + document: document("budget", vec![7; 64]), + }; + let encoded = encode_cultnet_message_to_vec(&put("put-a"), CultNetWireContract::CultNetSchemaV0)?; + let options = CultMeshRudpDocumentServerOptions { + max_admitted_payload_bytes: encoded.len() * 2, + max_admitted_payload_bytes_per_session: encoded.len(), + max_snapshot_response_bytes: encoded.len(), + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(47_000), sink.clone(), Source::default())?; + let target = server.local_addr()?; + let shared = UdpSocket::bind("127.0.0.1:0")?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, target, 94, 50_000)?; + connect(&mut server, &mut [&mut first])?; + send(&mut first, &put("put-a"))?; + for _ in 0..20 { + server.poll_once()?; + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + + let mut restarted = client_on(twin, target, 94, 7)?; + connect(&mut server, &mut [&mut restarted])?; + send(&mut restarted, &put("put-b"))?; + for _ in 0..20 { + server.poll_once()?; + } + assert_eq!( + sink.0.lock().unwrap().receipts.len(), + 2, + "the restarted client was charged what the old session spent" + ); + Ok(()) +} + #[test] fn application_rejection_is_nonfatal_peer_scoped_and_unacknowledged() -> Result<()> { let clock = Clock::new(48_000); @@ -576,7 +675,7 @@ fn a_connect_storm_and_stray_frames_are_dropped_not_fatal() -> Result<()> { let session = |id| { CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 10, max_pending_reliable_packets: None, }) diff --git a/packages/cultmesh-ts/src/index.ts b/packages/cultmesh-ts/src/index.ts index ab99639d..b89cd823 100644 --- a/packages/cultmesh-ts/src/index.ts +++ b/packages/cultmesh-ts/src/index.ts @@ -5793,7 +5793,7 @@ export class CultMesh { if (packet.packetType === "connect") { const connectPayload = Uint8Array.from(packet.payload ?? []); - if (record && byteArraysEqual(record.connectPayload, connectPayload)) { + if (record?.session.connectRepeats(packet)) { socket.send( encodeRudpPacket(record.session.acceptConnect(packet, nowMs)), remote.port, @@ -5846,7 +5846,7 @@ export class CultMesh { } if (packet.reliable) { socket.send( - encodeRudpPacket(record.session.createAckFor(packet.sequence)), + encodeRudpPacket(record.session.createAckForReceived(packet.sequence)), record.remote.port, record.remote.address, ); @@ -6671,14 +6671,6 @@ function nonBlankOr(value?: string, fallback = ""): string { return value && value.trim() ? value : fallback; } -function byteArraysEqual(left: Uint8Array, right: Uint8Array): boolean { - if (left.byteLength !== right.byteLength) return false; - for (let index = 0; index < left.byteLength; index += 1) { - if (left[index] !== right[index]) return false; - } - return true; -} - function parseCultMeshLocalityKind( value: string | undefined, fallback: CultMeshLocalityKind, diff --git a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts index 5b44c84b..f1dd5d2d 100644 --- a/packages/cultmesh-ts/test/provider-rudp-transport.test.ts +++ b/packages/cultmesh-ts/test/provider-rudp-transport.test.ts @@ -8,6 +8,7 @@ import { decodeRudpPacket, encodeRudpPacket, type CultNetOperationRequestMessage, + type CultNetRudpPacket, } from "cultnet-ts"; import { @@ -320,6 +321,80 @@ test("a replaced RUDP generation drops operations queued behind an active handle } }); +test("a client restarted on the same address with the same Connect payload replaces its RUDP session, and a retransmitted Connect does not", async () => { + const connectionId = 0x43554c54; + let closedSessions = 0; + const server = CultMesh.createRudpDocumentServer("odin-restart-test", connectionId, { + bindPort: 0, + documents: new CultNetDocumentRegistry(), + onSessionClosed: () => { closedSessions += 1; }, + onError: () => undefined, + }); + await server.start(); + const socket = createSocket("udp4"); + const received: CultNetRudpPacket[] = []; + socket.on("message", wire => received.push(decodeRudpPacket(wire))); + const toServer = (packet: CultNetRudpPacket) => + socket.send(encodeRudpPacket(packet), server.bind.port, "127.0.0.1"); + const evidence = Buffer.from("same connect payload"); + try { + const first = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const connect = first.createConnect(Date.now(), evidence); + toServer(connect); + await waitFor(() => received.some(packet => packet.packetType === "accept"), "the first Accept"); + + // A retransmitted Connect repeats: the session stays. + toServer(connect); + await new Promise(resolve => setTimeout(resolve, 100)); + assert.equal(closedSessions, 0, "a retransmitted Connect closed the session"); + + // The same client restarted: another sequence, the same payload. + received.length = 0; + const second = new CultNetRudpSession({ connectionId, initialSequence: 7 }); + toServer(second.createConnect(Date.now(), evidence)); + await waitFor(() => closedSessions === 1, "the old RUDP session to close"); + await waitFor(() => received.some(packet => packet.packetType === "accept"), "the restarted client's Accept"); + } finally { + socket.close(); + server.close(); + } +}); + +test("the RUDP document server does not acknowledge by name a reliable packet its session did not receive", async () => { + const connectionId = 0x43554c55; + const server = CultMesh.createRudpDocumentServer("odin-ack-test", connectionId, { + bindPort: 0, + documents: new CultNetDocumentRegistry(), + onError: () => undefined, + }); + await server.start(); + const socket = createSocket("udp4"); + const received: CultNetRudpPacket[] = []; + socket.on("message", wire => received.push(decodeRudpPacket(wire))); + const toServer = (packet: CultNetRudpPacket) => + socket.send(encodeRudpPacket(packet), server.bind.port, "127.0.0.1"); + try { + const client = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const connect = client.createConnect(Date.now()); + toServer(connect); + await waitFor(() => received.some(packet => packet.packetType === "accept"), "the Accept"); + received.length = 0; + + // A reliable packet the session ignores (a client does not send an Accept). + // The session never received it, so the Ack must not name it. + const ignored: CultNetRudpPacket = { ...client.createAck(), packetType: "accept", reliable: true, sequence: connect.sequence + 5 }; + toServer(ignored); + await waitFor(() => received.some(packet => packet.packetType === "ack"), "the acknowledgement"); + const ack = received.find(packet => packet.packetType === "ack")!; + const names = ack.ack === ignored.sequence + || Array.from({ length: 32 }, (_, bit) => bit).some(bit => (ack.ackMask & (1 << bit)) !== 0 && ack.ack - bit - 1 === ignored.sequence); + assert.equal(names, false, "the server acknowledged by name a packet its session refused"); + } finally { + socket.close(); + server.close(); + } +}); + function deferred(): { promise: Promise; resolve(value: T): void } { let resolve!: (value: T) => void; return { promise: new Promise(done => resolve = done), resolve }; diff --git a/packages/cultnet-py/src/cultnet_py/client.py b/packages/cultnet-py/src/cultnet_py/client.py index c30859d9..05bc5fa6 100644 --- a/packages/cultnet-py/src/cultnet_py/client.py +++ b/packages/cultnet-py/src/cultnet_py/client.py @@ -393,7 +393,7 @@ def create_rudp_schema_transport( runtime_id: str = "cultnet-python-rudp-client", bind_host: str = "127.0.0.1", bind_port: int = 0, - initial_sequence: int = 1, + initial_sequence: int | None = None, resend_delay_ms: int = 25, transport_id: str = "rudp", max_payload_bytes: int | None = None, diff --git a/packages/cultnet-py/src/cultnet_py/transport.py b/packages/cultnet-py/src/cultnet_py/transport.py index 0f378613..355faa51 100644 --- a/packages/cultnet-py/src/cultnet_py/transport.py +++ b/packages/cultnet-py/src/cultnet_py/transport.py @@ -1,5 +1,6 @@ from __future__ import annotations +import secrets import socket import threading import time @@ -292,10 +293,45 @@ class CultNetRudpReceiveResult: disconnect_reason: bytes = b"" +# How long a client keeps retransmitting a Connect nobody answered before it +# abandons that attempt for a fresh one. A Connect the server answers with an +# Ack, never an Accept, is one the server judged stale (see `accept_connect`); +# retransmitting the same sequence would never change its mind. +_RUDP_CONNECT_ATTEMPT_MS = 3_000 + + +def _draw_sequence() -> int: + """A sequence drawn from a secure source in [1, 2^31). The Connect's + sequence is what tells a peer whether a Connect repeats one it already + accepted or starts a new session, so two sessions must not share one by + default.""" + return 1 + secrets.randbelow(2**31 - 1) + + +def _at_or_before(sequence: int, mark: int, window: int) -> bool: + """Whether `sequence` is at or before `mark` in serial order, within the + receive window. The compare is modular, so it holds across the wrap of the + 32-bit space; a sequence further back than the window is the duplicate + test's below-window clause, and one ahead of the mark is never before it.""" + return ((mark - sequence) & 0xFFFFFFFF) < window + + +def _packet_acknowledges(packet: "CultNetRudpPacket", sequence: int) -> bool: + if packet.ack == sequence: + return True + return any( + packet.ack_mask & (1 << bit) and packet.ack > bit and packet.ack - bit - 1 == sequence + for bit in range(32) + ) + + @dataclass(frozen=True) class CultNetRudpSessionOptions: connection_id: int - initial_sequence: int = 1 + # The first sequence this session issues. Unset, the session draws its own + # at random: every session is a new identity to its peer, and an options + # object reused for several must not give them one sequence. + initial_sequence: int | None = None resend_delay_ms: int = 250 max_pending_reliable_packets: int | None = None @@ -320,7 +356,7 @@ class CultNetRudpSocketTransportOptions: mode: CultNetRudpSocketMode connection_id: int remote_addr: tuple[str, int] | None = None - initial_sequence: int = 1 + initial_sequence: int | None = None resend_delay_ms: int = 250 transport_id: str = "rudp" max_payload_bytes: int | None = None @@ -409,20 +445,49 @@ class CultNetRudpSession: if options.max_pending_reliable_packets is not None and options.max_pending_reliable_packets <= 0: raise ValueError("RUDP max_pending_reliable_packets must be greater than zero") self.max_pending_reliable_packets = options.max_pending_reliable_packets - self._next_sequence = _uint32(options.initial_sequence, "initial_sequence") + self._next_sequence = _uint32( + _draw_sequence() if options.initial_sequence is None else options.initial_sequence, + "initial_sequence", + ) if self._next_sequence == 0xFFFFFFFF: raise ValueError("RUDP initial_sequence must leave room for a reliable packet") self._next_sequenced_by_channel: dict[str, int] = {} self._next_fragment_id = 1 self._connected = False + # Advances every time a generation ends. Everything issued in a + # generation (writes, flushes) belongs to it and dies with it. + self._generation = 0 + # True from the end of a generation until the next Connect or Accept + # begins one. A flush started in that interval has no live generation + # to wait on. + self._ended = False + # The sequence of the Connect that started the current generation: sent + # by this side, or accepted from the peer. A Connect repeats exactly when + # the session is connected and the Connect carries this sequence. + self._connect_sequence: int | None = None + # A Connect this side sent is unanswered. Only then is an Accept honoured. + self._awaiting_accept = False + # When the unanswered Connect first went out; `due_resends` abandons it + # for a fresh attempt once _RUDP_CONNECT_ATTEMPT_MS has passed. + self._connect_started_at_ms = 0 + # The payload of the Connect this side sent, kept so a fresh attempt can + # always be built whatever became of the pending packet. + self._connect_payload = b"" self._last_received_at_ms: int | None = None self._highest_received_sequence: int | None = None self._received_sequences: set[int] = set() self._latest_sequenced_by_channel: dict[str, int] = {} self._pending_reliable: dict[int, _PendingReliablePacket] = {} self._queued_reliable: deque[CultNetRudpPacket] = deque() - self._ordered_next_sequence_by_channel: dict[str, int] = {} - self._ordered_buffers: dict[str, dict[int, tuple[CultNetRudpDeliveredFrame, int]]] = {} + # Every reliable sequence up to and including this one has been received + # since the peer state was last reset. Only the handshake seeds it: the + # peer's Connect on the accepting side, the peer's Accept on the + # connecting side. Reliable data that arrives before that is refused. + self._received_through: int | None = None + # Ordered frames received but not yet deliverable, keyed by first + # sequence. A frame is held for exactly one reason: a reliable sequence + # below it has not arrived. + self._ordered_held: dict[int, CultNetRudpDeliveredFrame] = {} self._fragment_buffers: dict[tuple[str, int], dict[str, Any]] = {} # Matches cultnet-rs max_pending_fragment_sets. A set that loses one # fragment is never completed; without a bound it is kept for the life @@ -455,23 +520,49 @@ class CultNetRudpSession: def outstanding_reliable_packet_count(self) -> int: return len(self._pending_reliable) + len(self._queued_reliable) - def reset_peer_state(self) -> None: - """Forgets everything learned from the peer; sequence numbers already issued stay issued.""" + @property + def generation(self) -> int: + return self._generation + + @property + def ended(self) -> bool: + return self._ended + + def _end_session(self) -> None: + """The one way a session generation ends. The session stops being + connected and what it still owed the peer dies with it: a write not yet + acknowledged is dropped, so no later session retransmits it or credits + an ack to it. What was learned from the peer is not touched: the peer + may not know the session ended, and forgetting what it sent would let its + retransmits be delivered twice.""" self._connected = False + self._ended = True + self._awaiting_accept = False + self._generation += 1 + self._pending_reliable.clear() + self._queued_reliable.clear() + + def reset_peer_state(self) -> None: + """Ends the current generation and forgets everything learned from the peer; sequence numbers already issued stay issued.""" + self._end_session() self._last_received_at_ms = None self._highest_received_sequence = None self._received_sequences.clear() self._next_sequenced_by_channel.clear() self._latest_sequenced_by_channel.clear() - self._pending_reliable.clear() - self._queued_reliable.clear() - self._ordered_next_sequence_by_channel.clear() - self._ordered_buffers.clear() + self._received_through = None + self._ordered_held.clear() self._fragment_buffers.clear() self._fragment_sets_evicted = 0 def create_connect(self, now_ms: int = 0, payload: bytes = b"") -> CultNetRudpPacket: + # A session that has had a peer starts a new generation: nothing it + # learned from that peer describes the one this Connect reaches, and + # what it still owed that peer no longer takes room in the queue. + if self._connect_sequence is not None: + self.reset_peer_state() self._ensure_reliable_capacity(1) + self._ended = False packet = self._create_packet( CultNetRudpPacketType.CONNECT, "control", @@ -479,22 +570,75 @@ class CultNetRudpSession: reliable=True, ordered=True, ) + self._connect_sequence = packet.sequence + self._awaiting_accept = True + self._connect_started_at_ms = now_ms + self._connect_payload = packet.payload self._track_reliable(packet, now_ms) return packet + def connect_repeats(self, packet: CultNetRudpPacket) -> bool: + """Whether `packet` is a Connect this session's current generation + already owns: a retransmit of the Connect that started it (the session + is connected and the sequence is that Connect's), or a stale copy of an + earlier attempt by the same client (a sequence before it, within the + receive window). Servers that keep one session per peer ask this to tell + a Connect that starts a new session from one that does not; anything + else that reaches `accept_connect` starts a new generation.""" + return ( + packet.packet_type == CultNetRudpPacketType.CONNECT + and self._connected + and self._connect_sequence is not None + and ( + self._connect_sequence == packet.sequence + or self._connect_is_stale(packet.sequence, self._connect_sequence) + ) + ) + + def _connect_is_stale(self, sequence: int, current: int) -> bool: + """A Connect that precedes the current generation's within the receive + window is the client's earlier attempt, delayed in the network: a client + that retried never sends a lower sequence again. Restarting on it would + strand the client, which honours only the Accept for its newest Connect. + The rule is TCP's answer to a delayed SYN (RFC 5961's challenge ACK): + keep the connection and answer with an Ack. A restarted client whose + random initial sequence lands in this window is answered the same way + and abandons the attempt for a fresh draw (_RUDP_CONNECT_ATTEMPT_MS).""" + return sequence != current and _at_or_before(sequence, current, self.RECEIVED_SEQUENCE_WINDOW) + def accept_connect( self, packet: CultNetRudpPacket, now_ms: int = 0, payload: bytes = b"", ) -> CultNetRudpPacket: + """Answers a Connect. A repeat of the accepted one queues nothing, so a + Connect storm cannot grow the reliable queue: the reply is the Accept + still awaiting acknowledgement, or an Ack once it was acknowledged. A + stale copy of an earlier attempt gets the same reply and changes nothing + else: it is not evidence the peer is alive. Any other Connect ends the + current generation, forgets the peer and accepts a new one.""" self._require_connection(packet) if packet.packet_type != CultNetRudpPacketType.CONNECT: raise ValueError(f"Expected RUDP connect packet, got {packet.packet_type.value}") + if self.connect_repeats(packet): + if self._connect_sequence == packet.sequence: + self._apply_acknowledgements(packet) + self._remember_received(packet.sequence) + self._last_received_at_ms = now_ms + for pending in self._pending_reliable.values(): + if pending.packet.packet_type == CultNetRudpPacketType.ACCEPT: + pending.last_sent_at_ms = now_ms + return pending.packet + return self.create_ack() + self.reset_peer_state() self._ensure_reliable_capacity(1) - self._remember_received(packet.sequence) + self._seed_received(packet.sequence) + self._last_received_at_ms = now_ms + self._connect_sequence = packet.sequence self._connected = True + self._ended = False response = self._create_packet( CultNetRudpPacketType.ACCEPT, "control", @@ -505,25 +649,6 @@ class CultNetRudpSession: self._track_reliable(response, now_ms) return response - def answer_repeated_connect(self, packet: CultNetRudpPacket, now_ms: int = 0) -> CultNetRudpPacket: - """Answers a Connect from a peer this session already accepted. - - The packet's sequence is remembered and nothing is queued, so a Connect - storm cannot grow the reliable queue. The reply is the Accept still awaiting - acknowledgement, or an Ack once it was acknowledged. - """ - self._require_connection(packet) - if packet.packet_type != CultNetRudpPacketType.CONNECT: - raise ValueError(f"Expected RUDP connect packet, got {packet.packet_type.value}") - self._apply_acknowledgements(packet) - self._remember_received(packet.sequence) - self._last_received_at_ms = now_ms - for pending in self._pending_reliable.values(): - if pending.packet.packet_type == CultNetRudpPacketType.ACCEPT: - pending.last_sent_at_ms = now_ms - return pending.packet - return self.create_ack() - def send( self, channel_id: str, @@ -589,17 +714,28 @@ class CultNetRudpSession: def receive(self, packet: CultNetRudpPacket, now_ms: int = 0) -> CultNetRudpReceiveResult: self._require_connection(packet) - self._apply_acknowledgements(packet) + # An Accept counts only while this side's Connect is unanswered and the + # Accept names it. A late or duplicate one, or one from an earlier + # generation, must not seed the watermark or revive an ended session. + honours_accept = ( + packet.packet_type == CultNetRudpPacketType.ACCEPT + and self._awaiting_accept + and self._connect_sequence is not None + and _packet_acknowledges(packet, self._connect_sequence) + ) + if packet.packet_type == CultNetRudpPacketType.ACCEPT and not honours_accept: + return CultNetRudpReceiveResult() + # While this side's Connect awaits its Accept, only the Accept it + # honours retires it: an Ack that names the Connect (a server's reply to + # a repeat or a stale copy) says the server did not start a session. + if honours_accept or not self._awaiting_accept: + self._apply_acknowledgements(packet) ready_to_send = self._promote_queued_reliable(now_ms) self._last_received_at_ms = now_ms - expected_sequence_if_uninitialized = ( - packet.sequence - if self._highest_received_sequence is None - else self._highest_received_sequence + 1 - ) - if packet.packet_type == CultNetRudpPacketType.ACCEPT: - self._remember_received(packet.sequence) + if honours_accept: + self._awaiting_accept = False + self._seed_received(packet.sequence) self._connected = True return CultNetRudpReceiveResult(ready_to_send=ready_to_send) @@ -617,7 +753,7 @@ class CultNetRudpSession: ) if packet.packet_type == CultNetRudpPacketType.DISCONNECT: - self._connected = False + self._end_session() return CultNetRudpReceiveResult( ready_to_send=ready_to_send, disconnected=True, @@ -627,37 +763,38 @@ class CultNetRudpSession: if packet.packet_type != CultNetRudpPacketType.DATA: return CultNetRudpReceiveResult(ready_to_send=ready_to_send) - duplicate = packet.reliable and ( - packet.sequence in self._received_sequences or ( - self._highest_received_sequence is not None - and packet.sequence < self._highest_received_sequence - and self._highest_received_sequence - packet.sequence >= self.RECEIVED_SEQUENCE_WINDOW - ) - ) + # Reliable data before the handshake has seeded the watermark has no place + # in the order: refuse it unremembered, so it is not acknowledged and the + # sender retransmits it once the handshake is done. + if packet.reliable and self._received_through is None: + return CultNetRudpReceiveResult(ready_to_send=ready_to_send) + + duplicate = packet.reliable and self._was_received(packet.sequence) if packet.reliable: self._remember_received(packet.sequence) if duplicate: return CultNetRudpReceiveResult(ready_to_send=ready_to_send) + delivered: list[CultNetRudpDeliveredFrame] = [] reassembled = self._reassemble(packet) - if reassembled is None: - return CultNetRudpReceiveResult(ready_to_send=ready_to_send) - frame, ordered, next_sequence = reassembled - if not ordered and packet.sequenced: - newest_sequence = next_sequence - 1 - latest_sequence = self._latest_sequenced_by_channel.get(frame.channel_id) - if latest_sequence is not None and newest_sequence <= latest_sequence: - return CultNetRudpReceiveResult(ready_to_send=ready_to_send) - self._latest_sequenced_by_channel[frame.channel_id] = newest_sequence - return CultNetRudpReceiveResult(delivered=(frame,), ready_to_send=ready_to_send) - if not ordered: - return CultNetRudpReceiveResult(delivered=(frame,), ready_to_send=ready_to_send) - return CultNetRudpReceiveResult( - delivered=tuple( - self._deliver_ordered(frame, next_sequence, expected_sequence_if_uninitialized) - ), - ready_to_send=ready_to_send, - ) + if reassembled is not None: + frame, ordered, next_sequence = reassembled + if ordered: + self._ordered_held[frame.sequence] = frame + elif not packet.sequenced: + delivered.append(frame) + else: + newest_sequence = next_sequence - 1 + latest_sequence = self._latest_sequenced_by_channel.get(frame.channel_id) + if latest_sequence is None or newest_sequence > latest_sequence: + self._latest_sequenced_by_channel[frame.channel_id] = newest_sequence + delivered.append(frame) + + # Any reliable packet may have advanced the watermark, a fragment or an + # unordered frame as much as an ordered one, so the drain runs after all + # of them. + delivered.extend(self._drain_ordered()) + return CultNetRudpReceiveResult(delivered=tuple(delivered), ready_to_send=ready_to_send) def create_ack(self) -> CultNetRudpPacket: ack, ack_mask = self._ack_state() @@ -683,7 +820,11 @@ class CultNetRudpSession: ) def create_ack_for_received(self, sequence: int) -> CultNetRudpPacket: + # What `receive` refused is not acknowledged by name (the ack carries only + # what was received), so its sender retransmits it. received_sequence = _uint32(sequence, "received sequence") + if not self._was_received(received_sequence): + return self.create_ack() ack, _ = self._ack_state() if ack >= received_sequence and ack - received_sequence <= 32: return self.create_ack() @@ -693,7 +834,7 @@ class CultNetRudpSession: return self._create_packet(CultNetRudpPacketType.PING, "control", payload) def create_disconnect(self, reason: bytes = b"") -> CultNetRudpPacket: - self._connected = False + self._end_session() return self._create_packet(CultNetRudpPacketType.DISCONNECT, "control", reason) def check_timeout(self, now_ms: int, timeout_ms: int) -> bool: @@ -701,10 +842,13 @@ class CultNetRudpSession: return False if now_ms - self._last_received_at_ms <= timeout_ms: return False - self._connected = False + self._end_session() return True def due_resends(self, now_ms: int) -> tuple[CultNetRudpPacket, ...]: + fresh = self._abandon_unanswered_connect(now_ms) + if fresh is not None: + return (fresh,) due: list[CultNetRudpPacket] = [] for pending in self._pending_reliable.values(): if now_ms - pending.last_sent_at_ms >= self.resend_delay_ms: @@ -712,6 +856,24 @@ class CultNetRudpSession: due.append(pending.packet) return tuple(sorted(due, key=lambda packet: packet.sequence)) + def _abandon_unanswered_connect(self, now_ms: int) -> CultNetRudpPacket | None: + """A Connect unanswered for _RUDP_CONNECT_ATTEMPT_MS is replaced, not + retransmitted further, by a Connect with the same payload and the + abandoned sequence plus the receive window less one. Nothing was ever + sent in an unanswered generation, so no sequence issued so far is owed. + The jump keeps a late copy of the abandoned Connect inside the new one's + stale window, so a server that took the new Connect answers the copy + with an Ack instead of restarting; and it leaves the stale window of + whatever generation the server holds, unless that generation sits + exactly at the jump, and then the next attempt leaves it. The first + Connect of a session is the only one drawn at random.""" + if not self._awaiting_accept or now_ms - self._connect_started_at_ms < _RUDP_CONNECT_ATTEMPT_MS: + return None + if self._connect_sequence is None: + return None + self._next_sequence = min(self._connect_sequence + self.RECEIVED_SEQUENCE_WINDOW - 1, 0xFFFFFFFE) + return self.create_connect(now_ms, self._connect_payload) + def _create_packet( self, packet_type: CultNetRudpPacketType, @@ -806,8 +968,39 @@ class CultNetRudpSession: if packet.ack_mask & (1 << bit): self._pending_reliable.pop(packet.ack - bit - 1, None) + def _seed_received(self, sequence: int) -> None: + """The handshake's one act on the watermark: the peer's Connect or Accept + is the first sequence of the session, whatever else it has sent.""" + self._received_through = sequence + self._remember_received(sequence) + + def _was_received(self, sequence: int) -> bool: + """The one duplicate test: true for a sequence in the window that was + received, for one below the window, and for one at or before the + watermark. The watermark starts at the handshake's seed, so a frame the + peer sent in an earlier generation (every sequence it issued is below the + Connect that began this one) is a duplicate of something already + delivered and is acknowledged, never delivered again.""" + return ( + sequence in self._received_sequences + or ( + self._highest_received_sequence is not None + and sequence < self._highest_received_sequence + and self._highest_received_sequence - sequence >= self.RECEIVED_SEQUENCE_WINDOW + ) + or ( + self._received_through is not None + and _at_or_before(sequence, self._received_through, self.RECEIVED_SEQUENCE_WINDOW) + ) + ) + def _remember_received(self, sequence: int) -> None: self._received_sequences.add(sequence) + if self._received_through is not None: + through = self._received_through + while through + 1 in self._received_sequences: + through += 1 + self._received_through = through if self._highest_received_sequence is None or sequence > self._highest_received_sequence: self._highest_received_sequence = sequence if len(self._received_sequences) > self.RECEIVED_SEQUENCE_WINDOW: @@ -865,58 +1058,15 @@ class CultNetRudpSession: max(sequences) + 1, ) - def _deliver_ordered( - self, - frame: CultNetRudpDeliveredFrame, - next_after_frame: int, - expected_sequence_if_uninitialized: int, - ) -> list[CultNetRudpDeliveredFrame]: - next_sequence = self._ordered_next_sequence_by_channel.get(frame.channel_id) - if next_sequence is None: - next_sequence = min(expected_sequence_if_uninitialized, frame.sequence) - self._ordered_next_sequence_by_channel[frame.channel_id] = next_sequence - while ( - frame.sequence > next_sequence - and next_sequence in self._received_sequences - and next_sequence not in self._ordered_buffers.get(frame.channel_id, {}) - ): - next_sequence += 1 - self._ordered_next_sequence_by_channel[frame.channel_id] = next_sequence - if frame.sequence < next_sequence: + def _drain_ordered(self) -> list[CultNetRudpDeliveredFrame]: + """Delivers, in sequence order, every held ordered frame whose first + sequence is at most one past the watermark. Ordered delivery has one + owner: the watermark. A frame this call does not deliver waits for the + sequence below it, and nothing else releases it.""" + if self._received_through is None: return [] - if frame.sequence > next_sequence: - self._ordered_buffers.setdefault(frame.channel_id, {})[frame.sequence] = (frame, next_after_frame) - return [] - - self._ordered_next_sequence_by_channel[frame.channel_id] = next_after_frame - return [frame, *self._drain_ordered(frame.channel_id)] - - def _drain_ordered(self, channel_id: str) -> list[CultNetRudpDeliveredFrame]: - delivered: list[CultNetRudpDeliveredFrame] = [] - buffer = self._ordered_buffers.get(channel_id) - if buffer is None: - return delivered - - while True: - next_sequence = self._ordered_next_sequence_by_channel[channel_id] - pending = buffer.pop(next_sequence, None) - if pending is None: - break - frame, next_after_frame = pending - delivered.append(frame) - self._ordered_next_sequence_by_channel[channel_id] = next_after_frame - self._skip_received_non_channel_sequences(channel_id) - return delivered - - def _skip_received_non_channel_sequences(self, channel_id: str) -> None: - next_sequence = self._ordered_next_sequence_by_channel.get(channel_id) - while ( - next_sequence is not None - and next_sequence in self._received_sequences - and next_sequence not in self._ordered_buffers.get(channel_id, {}) - ): - next_sequence += 1 - self._ordered_next_sequence_by_channel[channel_id] = next_sequence + releasable = sorted(sequence for sequence in self._ordered_held if sequence <= self._received_through + 1) + return [self._ordered_held.pop(sequence) for sequence in releasable] def _allocate_fragment_id(self) -> int: fragment_id = self._next_fragment_id @@ -1033,6 +1183,10 @@ class CultNetRudpSocketTransportConnection: if packet.connection_id != self.session.connection_id: self._packets_dropped += 1 return None + # A Connect from another endpoint is a new client, whatever sequence it + # carries: the endpoint moves and the Connect starts a new generation. + # Only a Connect from the peer's own endpoint can be its repeat. + connect_from_new_endpoint = False if self.remote_addr is None: # Only a Connect claims the endpoint of a server-mode transport. if self.mode == CultNetRudpSocketMode.SERVER and packet.packet_type != CultNetRudpPacketType.CONNECT: @@ -1040,18 +1194,21 @@ class CultNetRudpSocketTransportConnection: return None self.remote_addr = remote_addr elif remote_addr != self.remote_addr: - self._packets_dropped += 1 - return None + if self.mode == CultNetRudpSocketMode.SERVER and packet.packet_type == CultNetRudpPacketType.CONNECT: + self.remote_addr = remote_addr + connect_from_new_endpoint = True + else: + self._packets_dropped += 1 + return None if self.mode == CultNetRudpSocketMode.SERVER and packet.packet_type == CultNetRudpPacketType.CONNECT: - # A Connect from the peer this transport already accepted repeats: - # answer it with the Accept already owed and queue nothing. + # The session answers a repeat of the Connect it accepted with the + # Accept already owed, and anything else with a new generation. try: - if self.session.connected: - reply = self.session.answer_repeated_connect(packet, _now_ms()) - else: - reply = self.session.accept_connect(packet, _now_ms()) - self.disconnect_reason = None + if connect_from_new_endpoint: + self.session.reset_peer_state() + reply = self.session.accept_connect(packet, _now_ms()) + self.disconnect_reason = None except ValueError: self._packets_dropped += 1 return None @@ -1105,14 +1262,19 @@ class CultNetRudpSocketTransportConnection: preserved = deque(self._delivered_frames) self._delivered_frames.clear() self.socket.settimeout(poll_timeout) + # A flush belongs to the generation it started in: whatever ends that + # generation, and whatever begins after it, the writes being waited on + # are gone. + generation = self.session.generation try: while True: # An ended session forgot its unacknowledged writes; reporting # them flushed would be a lie. - if self.disconnect_reason is not None: + if self.session.ended or self.session.generation != generation: + reason = self.disconnect_reason raise ConnectionError( "RUDP session ended before its reliable writes were acknowledged: " - + self.disconnect_reason.decode("utf-8", errors="replace") + + ("no reason given" if reason is None else reason.decode("utf-8", errors="replace")) ) if self.session.outstanding_reliable_packet_count == 0: return @@ -1137,7 +1299,10 @@ class CultNetRudpSocketTransportConnection: self._send_packet(self.session.create_ping(payload)) def check_timeout(self, timeout_ms: int, now_ms: int | None = None) -> bool: - return self.session.check_timeout(_now_ms() if now_ms is None else now_ms, timeout_ms) + timed_out = self.session.check_timeout(_now_ms() if now_ms is None else now_ms, timeout_ms) + if timed_out: + self.disconnect_reason = b"session timed out" + return timed_out def poll_resends(self) -> None: for packet in self.session.due_resends(_now_ms()): diff --git a/packages/cultnet-py/tests/test_cultnet.py b/packages/cultnet-py/tests/test_cultnet.py index 7e78c48f..fb2eff54 100644 --- a/packages/cultnet-py/tests/test_cultnet.py +++ b/packages/cultnet-py/tests/test_cultnet.py @@ -94,6 +94,17 @@ from cultnet_py import ( ) +def acknowledges(packet, sequence: int) -> bool: + """Whether the packet's ack field and mask cover the sequence.""" + if packet.ack == sequence: + return True + return ( + packet.ack > sequence + and packet.ack - sequence <= 32 + and (packet.ack_mask & (1 << (packet.ack - sequence - 1))) != 0 + ) + + def bind_udp_socket() -> socket.socket: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.bind(("127.0.0.1", 0)) @@ -616,12 +627,7 @@ class CultNetTests(unittest.TestCase): receiver = CultNetRudpSession( CultNetRudpSessionOptions(connection_id=7, initial_sequence=200, resend_delay_ms=100) ) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 7, 1, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 7, 2, 0, 0, "control") - ) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) first = sender.send( "schema", @@ -638,27 +644,28 @@ class CultNetTests(unittest.TestCase): b"third", CultNetRudpSendOptions(reliable=True, ordered=True, now_ms=0), ) - self.assertEqual(sender.pending_reliable_sequences, (10, 11, 12)) + self.assertEqual(sender.pending_reliable_sequences, (11, 12, 13)) receiver.receive(first) receiver.receive(third) ack_with_gap = receiver.create_ack() - self.assertEqual(ack_with_gap.ack, 12) - self.assertEqual(ack_with_gap.ack_mask, 0b10 | (1 << 9)) + self.assertEqual(ack_with_gap.ack, 13) + self.assertEqual(ack_with_gap.ack_mask, 0b110) sender.receive(ack_with_gap) - self.assertEqual(sender.pending_reliable_sequences, (11,)) + self.assertEqual(sender.pending_reliable_sequences, (12,)) receiver.receive(second) full_ack = receiver.create_ack() - self.assertEqual(full_ack.ack, 12) - self.assertEqual(full_ack.ack_mask, 0b11 | (1 << 9)) + self.assertEqual(full_ack.ack, 13) + self.assertEqual(full_ack.ack_mask, 0b111) sender.receive(full_ack) self.assertEqual(sender.pending_reliable_sequences, ()) def test_cultnet_rudp_session_schedules_reliable_resends_until_acked(self) -> None: session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=99, initial_sequence=1, resend_delay_ms=100)) session.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 99, 50, 0, 0, "control") + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=99, initial_sequence=900)).accept_connect(session.create_connect(0), 0), + 0, ) sent = session.send( "schema", @@ -700,7 +707,7 @@ class CultNetTests(unittest.TestCase): server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=103, initial_sequence=100)) connect = client.create_connect(0, b"join") accept = server.accept_connect(connect, 0) - reply = server.answer_repeated_connect(replace(connect, ack=accept.sequence), 1) + reply = server.accept_connect(replace(connect, ack=accept.sequence), 1) self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) self.assertEqual(server.outstanding_reliable_packet_count, 0) @@ -708,17 +715,23 @@ class CultNetTests(unittest.TestCase): session = CultNetRudpSession( CultNetRudpSessionOptions(connection_id=102, initial_sequence=1, max_pending_reliable_packets=2) ) - session.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 102, 50, 0, 0, "control")) + session.receive( + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=102, initial_sequence=900)).accept_connect(session.create_connect(0), 0), + 0, + ) session.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) session.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) with self.assertRaisesRegex(ValueError, "reliable send queue is full"): session.send("schema", b"third", CultNetRudpSendOptions(reliable=True, ordered=True)) - self.assertEqual(session.pending_reliable_sequences, (1, 2)) + self.assertEqual(session.pending_reliable_sequences, (2, 3)) fragmented = CultNetRudpSession( CultNetRudpSessionOptions(connection_id=103, initial_sequence=1, max_pending_reliable_packets=3) ) - fragmented.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 103, 50, 0, 0, "control")) + fragmented.receive( + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=103, initial_sequence=900)).accept_connect(fragmented.create_connect(0), 0), + 0, + ) with self.assertRaisesRegex(ValueError, "reliable send queue is full"): fragmented.send_many( "schema", @@ -731,12 +744,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_session_suppresses_duplicates_and_delivers_reliable_ordered_payloads(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=123, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=123, initial_sequence=100)) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 123, 90, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 123, 91, 0, 0, "control") - ) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0)) first = sender.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) second = sender.send("schema", b"second", CultNetRudpSendOptions(reliable=True, ordered=True)) @@ -753,8 +761,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_lossy_packets_cannot_create_reliable_ordered_gaps(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=198, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=198, initial_sequence=100)) - sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 198, 90, 0, 0, "control")) - receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 198, 91, 0, 0, "control")) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) realtime = sender.send("realtime", b"discarded realtime", CultNetRudpSendOptions()) latest = sender.send("latest", b"discarded latest state", CultNetRudpSendOptions(sequenced=True)) @@ -770,8 +777,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_unreliable_sequenced_delivery_is_scoped_to_its_channel(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=197, initial_sequence=50)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=197, initial_sequence=100)) - sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 197, 90, 0, 0, "control")) - receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 197, 91, 0, 0, "control")) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) older = sender.send("latest", b"older", CultNetRudpSendOptions(sequenced=True)) newer = sender.send("latest", b"newer", CultNetRudpSendOptions(sequenced=True)) @@ -786,7 +792,10 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_rejects_unreliable_ordered_delivery(self) -> None: session = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=196, initial_sequence=1)) - session.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 196, 50, 0, 0, "control")) + session.receive( + CultNetRudpSession(CultNetRudpSessionOptions(connection_id=196, initial_sequence=900)).accept_connect(session.create_connect(0), 0), + 0, + ) with self.assertRaises(ValueError) as raised: session.send( @@ -799,12 +808,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_session_skips_control_packets_while_ordering_schema_payloads(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=124, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=124, initial_sequence=100)) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 124, 90, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 124, 91, 0, 0, "control") - ) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) first = sender.send("schema", b"first", CultNetRudpSendOptions(reliable=True, ordered=True)) control = sender.create_ack() @@ -817,12 +821,7 @@ class CultNetTests(unittest.TestCase): def test_cultnet_rudp_session_fragments_and_reassembles_reliable_ordered_payloads(self) -> None: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=456, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=456, initial_sequence=100)) - sender.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 456, 90, 0, 0, "control") - ) - receiver.receive( - CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 456, 91, 0, 0, "control") - ) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) packets = sender.send_many( "schema", @@ -847,8 +846,7 @@ class CultNetTests(unittest.TestCase): connection_id = 457 sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=100)) - sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, connection_id, 0, 0, 0, "control")) - receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, connection_id, 0, 0, 0, "control")) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) fragment_count = CultNetRudpSession.RELIABLE_SEND_WINDOW_PACKETS + 17 payload = bytes(index % 251 for index in range(fragment_count * 8)) @@ -1068,14 +1066,13 @@ class CultNetTests(unittest.TestCase): # The peer is told, and the goodbye does not acknowledge the refused frame. (goodbye,) = [p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.DISCONNECT] self.assertTrue(peer.receive(goodbye, 1).disconnected) - self.assertIn(poison.sequence, peer.pending_reliable_sequences) + self.assertFalse(acknowledges(goodbye, poison.sequence), "the goodbye acknowledged the refused frame") # A retransmit is dropped, not acknowledged. to_server(poison) self.assertIsNone(server.receive_once()) self.assertEqual(server.stats.packets_dropped, 2) self.assertEqual(drain_peer(), []) - self.assertIn(poison.sequence, peer.pending_reliable_sequences) # A Connect claims the endpoint again. again = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id)) @@ -1145,6 +1142,135 @@ class CultNetTests(unittest.TestCase): peer_socket.close() server.close() + def test_cultnet_rudp_server_mode_admits_a_restarted_client_and_keeps_its_session_on_a_retransmit(self) -> None: + server, _, peer_socket, _, to_server, drain_peer = self._server_mode_pair(0x10203058) + connection_id = 0x10203058 + reliable = CultNetRudpSendOptions(reliable=True, ordered=True) + try: + first = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=50_000)) + connect = first.create_connect(0, b"same") + to_server(connect) + server.receive_once() + first.receive([p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0], 0) + sent = [first.send("schema", f"c{index}".encode(), reliable) for index in range(3)] + delivered = [] + for packet in sent: + to_server(packet) + delivered.append(server.receive_once().payload) + self.assertEqual(delivered, [b"c0", b"c1", b"c2"]) + + # A retransmitted Connect is a repeat: what was delivered stays delivered once. + to_server(connect) + server.receive_once() + to_server(sent[0]) + self.assertIsNone(server.receive_once(), "the retransmitted Connect reset the session") + + # The same client restarted on the same address, connection id and payload. + drain_peer() + second = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=7)) + to_server(second.create_connect(0, b"same")) + server.receive_once() + second.receive([p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0], 0) + to_server(second.send("schema", b"after restart", reliable)) + self.assertEqual(server.receive_once().payload, b"after restart") + finally: + peer_socket.close() + server.close() + + def test_cultnet_rudp_server_mode_admits_a_pinned_client_that_restarts_on_a_new_port(self) -> None: + server, server_socket, peer_socket, _, to_server, drain_peer = self._server_mode_pair(0x1020305B) + connection_id = 0x1020305B + reliable = CultNetRudpSendOptions(reliable=True, ordered=True) + other_socket = bind_udp_socket() + try: + first = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) + to_server(first.create_connect(0, b"join")) + server.receive_once() + first.receive([p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0], 0) + to_server(first.send("schema", b"hello", reliable)) + self.assertEqual(server.receive_once().payload, b"hello") + + # The process restarted on a new port with the same pinned sequence. + second = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) + other_socket.sendto(encode_rudp_packet(second.create_connect(0, b"join")), server_socket.getsockname()) + server.receive_once() + wire, _ = other_socket.recvfrom(65535) + second.receive(decode_rudp_packet(wire), 0) + self.assertTrue(second.connected, "the restarted client on a new port was not admitted") + other_socket.sendto(encode_rudp_packet(second.send("schema", b"after", reliable)), server_socket.getsockname()) + self.assertEqual(server.receive_once().payload, b"after") + finally: + other_socket.close() + peer_socket.close() + server.close() + + def test_cultnet_rudp_server_mode_admits_a_pinned_client_that_restarts_on_the_same_address(self) -> None: + server, _, peer_socket, _, to_server, drain_peer = self._server_mode_pair(0x1020305C) + connection_id = 0x1020305C + try: + first = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) + to_server(first.create_connect(0, b"join")) + server.receive_once() + accept = [p for p in drain_peer() if p.packet_type == CultNetRudpPacketType.ACCEPT][0] + first.receive(accept, 0) + to_server(first.create_ack_for_received(accept.sequence)) + server.receive_once() + drain_peer() + + # The process restarted on the same address with the same pinned sequence. + restarted = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=connection_id, initial_sequence=1)) + to_server(restarted.create_connect(0, b"join")) + server.receive_once() + for packet in drain_peer(): + restarted.receive(packet, 1) + self.assertFalse(restarted.connected) + + to_server(restarted.due_resends(3_000)[0]) + server.receive_once() + for packet in drain_peer(): + restarted.receive(packet, 3_001) + self.assertTrue(restarted.connected, "the restarted pinned client was never admitted") + finally: + peer_socket.close() + server.close() + + def test_cultnet_rudp_server_mode_admits_a_connect_from_a_new_endpoint(self) -> None: + server, server_socket, peer_socket, peer, to_server, drain_peer = self._server_mode_pair(0x10203059) + other_socket = bind_udp_socket() + try: + to_server(peer.create_connect(0, b"a")) + server.receive_once() + drain_peer() + + other = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=0x10203059, initial_sequence=9)) + other_socket.sendto(encode_rudp_packet(other.create_connect(0, b"b")), server_socket.getsockname()) + server.receive_once() + wire, _ = other_socket.recvfrom(65535) + other.receive(decode_rudp_packet(wire), 0) + self.assertTrue(other.connected, "the new endpoint was not admitted") + other_socket.sendto( + encode_rudp_packet(other.send("schema", b"from b", CultNetRudpSendOptions(reliable=True, ordered=True))), + server_socket.getsockname(), + ) + self.assertEqual(server.receive_once().payload, b"from b") + finally: + other_socket.close() + peer_socket.close() + server.close() + + def test_cultnet_rudp_flush_after_a_timeout_names_the_timeout(self) -> None: + server, _, peer_socket, peer, to_server, drain_peer = self._server_mode_pair(0x1020305A) + try: + to_server(peer.create_connect(0, b"join")) + server.receive_once() + server.send("schema", b"unanswered") + self.assertTrue(server.check_timeout(0, now_ms=10**14)) + with self.assertRaisesRegex(ConnectionError, "session timed out"): + server.flush_reliable(0.1) + finally: + peer_socket.close() + server.close() + def test_cultnet_rudp_session_rejects_limits_that_cannot_admit_a_connect(self) -> None: with self.assertRaises(ValueError): CultNetRudpSession(CultNetRudpSessionOptions(connection_id=1, initial_sequence=0xFFFFFFFF)) @@ -2277,8 +2403,7 @@ class CultNetRudpFragmentBoundTests(unittest.TestCase): def _pair(self) -> tuple[CultNetRudpSession, CultNetRudpSession]: sender = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=311, initial_sequence=1)) receiver = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=311, initial_sequence=100)) - sender.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 311, 90, 0, 0, "control")) - receiver.receive(CultNetRudpPacket(CultNetRudpPacketType.ACCEPT, 311, 91, 0, 0, "control")) + sender.receive(receiver.accept_connect(sender.create_connect(0), 0), 0) return sender, receiver def test_stranded_fragment_sets_are_bounded_and_evicted_oldest_first(self) -> None: diff --git a/packages/cultnet-py/tests/test_rudp_connect_generation.py b/packages/cultnet-py/tests/test_rudp_connect_generation.py new file mode 100644 index 00000000..4bbd896a --- /dev/null +++ b/packages/cultnet-py/tests/test_rudp_connect_generation.py @@ -0,0 +1,427 @@ +"""A Connect's sequence says whether it repeats the one a session accepted, and +the handshake alone seeds the watermark that orders delivery.""" +from __future__ import annotations + +import unittest +from dataclasses import replace + +from cultnet_py.transport import ( + CultNetRudpPacketType, + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, + CultNetRudpSocketMode, + CultNetRudpSocketTransportOptions, +) + +CONNECTION_ID = 0x10203090 + + +def session(initial_sequence: int) -> CultNetRudpSession: + return CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=initial_sequence)) + + +def send(from_session: CultNetRudpSession, text: str, *, ordered: bool = True, channel: str = "schema"): + return from_session.send(channel, text.encode(), CultNetRudpSendOptions(reliable=True, ordered=ordered)) + + +def names(result) -> list[str]: + return [frame.payload.decode() for frame in result.delivered] + + +def handshake(client: CultNetRudpSession, server: CultNetRudpSession): + connect = client.create_connect(0) + accept = server.accept_connect(connect, 0) + client.receive(accept, 0) + assert client.connected and server.connected + return connect, accept + + +class CultNetRudpConnectGenerationTests(unittest.TestCase): + def test_a_reconnect_on_the_same_session_is_not_stranded_by_a_write_the_ended_session_owed(self) -> None: + client, server = session(1), session(500) + handshake(client, server) + send(server, "lost") + server.create_disconnect(b"gone") + + client.receive(server.accept_connect(client.create_connect(1), 1), 1) + self.assertEqual(names(client.receive(send(server, "after"), 2)), ["after"]) + + def test_a_server_that_accepts_a_new_client_forgets_the_old_clients_sequences(self) -> None: + old_client, server = session(50), session(500) + handshake(old_client, server) + for name in ("c1", "c2", "c3"): + self.assertEqual(names(server.receive(send(old_client, name), 1)), [name]) + server.create_disconnect(b"restart") + + new_client = session(2) + new_client.receive(server.accept_connect(new_client.create_connect(2), 2), 2) + self.assertEqual(names(server.receive(send(new_client, "fresh"), 3)), ["fresh"]) + + def test_a_retransmitted_connect_after_data_has_flowed_does_not_reset_the_session(self) -> None: + client, server = session(1), session(500) + connect, _ = handshake(client, server) + frame = send(client, "once") + self.assertEqual(names(server.receive(frame, 1)), ["once"]) + + self.assertTrue(server.connect_repeats(connect)) + self.assertEqual(server.accept_connect(connect, 2).packet_type, CultNetRudpPacketType.ACK) + self.assertEqual(server.outstanding_reliable_packet_count, 0, "a repeat queued something") + self.assertEqual( + names(server.receive(frame, 3)), [], "a retransmitted Connect made the server forget what it had delivered" + ) + + def test_a_connect_with_another_sequence_is_not_a_repeat_whatever_it_carries(self) -> None: + server = session(500) + connect = session(1).create_connect(0, b"same payload") + server.accept_connect(connect, 0) + self.assertTrue(server.connect_repeats(connect)) + + self.assertFalse(server.connect_repeats(session(9).create_connect(0, b"same payload"))) + self.assertFalse(server.connect_repeats(replace(connect, packet_type=CultNetRudpPacketType.PING))) + server.create_disconnect() + self.assertFalse(server.connect_repeats(connect), "an ended session repeats nothing") + + def test_an_accept_that_does_not_name_the_pending_connect_is_ignored(self) -> None: + client, server = session(10), session(500) + connect = client.create_connect(0) + accept = server.accept_connect(connect, 0) + + client.receive(replace(accept, ack=connect.sequence + 50, ack_mask=0), 1) + self.assertFalse(client.connected, "a stale Accept connected the session") + self.assertEqual(client.pending_reliable_sequences, (connect.sequence,)) + + client.receive(replace(accept, ack=connect.sequence + 3, ack_mask=1 << 2), 2) + self.assertTrue(client.connected, "the mask names the Connect") + + def test_an_accept_after_a_local_disconnect_does_not_reconnect_the_session(self) -> None: + client, server = session(1), session(500) + _, accept = handshake(client, server) + client.create_disconnect(b"bye") + self.assertFalse(client.connected) + + client.receive(accept, 5) + self.assertFalse(client.connected, "a late Accept revived an ended session") + + def test_a_reconnect_forgets_what_the_old_server_sent(self) -> None: + client, old_server = session(1), session(500) + handshake(client, old_server) + for name in ("d1", "d2", "d3"): + self.assertEqual(names(client.receive(send(old_server, name), 1)), [name]) + + new_server = session(501) + client.receive(new_server.accept_connect(client.create_connect(2), 2), 2) + self.assertEqual(names(client.receive(send(new_server, "fresh"), 3)), ["fresh"]) + + def test_an_accept_for_an_abandoned_connect_does_not_connect_the_session(self) -> None: + client, server = session(1), session(500) + accept = server.accept_connect(client.create_connect(0), 0) + client.create_disconnect(b"never mind") + + client.receive(accept, 1) + self.assertFalse(client.connected, "an Accept revived an abandoned Connect") + + def test_a_full_queue_owed_to_a_vanished_peer_does_not_refuse_a_new_generation(self) -> None: + def bounded(initial_sequence: int) -> CultNetRudpSession: + return CultNetRudpSession( + CultNetRudpSessionOptions( + connection_id=CONNECTION_ID, initial_sequence=initial_sequence, max_pending_reliable_packets=1 + ) + ) + + server = bounded(500) + server.accept_connect(session(1).create_connect(0), 0) + self.assertEqual(server.outstanding_reliable_packet_count, 1, "the Accept is owed and never acknowledged") + following = session(9).create_connect(1) + server.accept_connect(following, 1) + self.assertTrue(server.connect_repeats(following)) + + client = bounded(1) + client.create_connect(0) + client.create_connect(1) + self.assertEqual(client.outstanding_reliable_packet_count, 1, "only the new Connect is owed") + + def test_a_duplicate_accept_does_not_reseed_the_watermark(self) -> None: + client, server = session(1), session(500) + _, accept = handshake(client, server) + s1, s2 = send(server, "s1"), send(server, "s2") + client.receive(replace(accept, sequence=s2.sequence), 1) + self.assertEqual(names(client.receive(s2, 2)), [], "s1 is still missing") + self.assertEqual(names(client.receive(s1, 3)), ["s1", "s2"]) + + def test_reliable_data_before_the_accept_is_neither_delivered_nor_acknowledged(self) -> None: + client, server = session(1), session(500) + accept = server.accept_connect(client.create_connect(0), 0) + a, b = send(server, "A"), send(server, "B") + + for early in (b, a): + self.assertEqual(names(client.receive(early, 1)), []) + ack = client.create_ack_for_received(early.sequence) + self.assertEqual((ack.ack, ack.ack_mask), (0, 0), "an unhandled packet was acknowledged") + client.receive(accept, 2) + self.assertTrue(client.connected) + self.assertEqual(server.outstanding_reliable_packet_count, 3, "nothing was acknowledged") + + self.assertEqual(names(client.receive(a, 3)), ["A"]) + self.assertEqual(names(client.receive(b, 3)), ["B"]) + + def test_ordered_frames_after_the_accept_are_delivered_in_order_whatever_arrives_first(self) -> None: + client, server = session(1), session(500) + handshake(client, server) + a, b = send(server, "A"), send(server, "B") + self.assertEqual(names(client.receive(b, 1)), []) + self.assertEqual(names(client.receive(a, 2)), ["A", "B"]) + + def test_an_unordered_reliable_frame_is_delivered_on_receipt_never_held_behind_a_gap(self) -> None: + client, server = session(1), session(500) + handshake(client, server) + send(server, "lost") + unordered = send(server, "now", ordered=False, channel="media") + ordered = send(server, "later") + self.assertEqual(names(client.receive(unordered, 1)), ["now"]) + self.assertEqual(names(client.receive(ordered, 2)), [], "the ordered frame waits for the gap") + + def test_a_peer_disconnect_does_not_forget_what_the_peer_sent(self) -> None: + client, server = session(1), session(500) + handshake(client, server) + s1, s2 = send(client, "s1"), send(client, "s2") + self.assertEqual(names(server.receive(s2, 1)), [], "s1 is missing, so s2 is held") + + server.receive(client.create_disconnect(b"bye"), 2) + self.assertEqual(names(server.receive(s1, 3)), ["s1", "s2"], "the Disconnect forgot the held frame") + self.assertEqual(names(server.receive(s2, 4)), [], "the Disconnect forgot what was received") + + def test_sessions_from_one_options_object_each_draw_their_own_initial_sequence(self) -> None: + self.assertIsNone(CultNetRudpSessionOptions(connection_id=1).initial_sequence) + self.assertIsNone( + CultNetRudpSocketTransportOptions( + runtime_id="r", socket=None, mode=CultNetRudpSocketMode.CLIENT, connection_id=1 + ).initial_sequence + ) + options = CultNetRudpSessionOptions(connection_id=1) + firsts = {CultNetRudpSession(options).create_connect(0).sequence for _ in range(64)} + self.assertGreater(len(firsts), 32, "a default session does not draw its own sequence") + self.assertTrue(all(1 <= value < 2**31 for value in firsts), "a draw left [1, 2^31)") + + # Ack Cut 1d: a frame from an earlier generation is a duplicate, a delayed + # Connect from an earlier attempt is stale, a Connect from a new endpoint is + # a new client. + + @staticmethod + def names_sequence(ack, sequence: int) -> bool: + return ack.ack == sequence or any( + ack.ack_mask & (1 << bit) and ack.ack > bit and ack.ack - bit - 1 == sequence for bit in range(32) + ) + + def test_a_frame_from_the_clients_earlier_generation_is_acknowledged_not_delivered_again(self) -> None: + client, server = session(10), session(500) + handshake(client, server) + old = send(client, "old") + self.assertEqual(names(server.receive(old, 1)), ["old"]) + + connect = client.create_connect(2) + client.receive(server.accept_connect(connect, 2), 2) + self.assertLess(old.sequence, connect.sequence) + self.assertEqual(names(server.receive(old, 3)), [], "an earlier generation's frame was delivered again") + self.assertTrue(self.names_sequence(server.create_ack_for_received(old.sequence), old.sequence)) + self.assertEqual(names(server.receive(send(client, "fresh"), 4)), ["fresh"]) + + def test_a_frame_from_the_servers_earlier_generation_is_acknowledged_not_delivered_again(self) -> None: + client, server = session(10), session(500) + handshake(client, server) + old = send(server, "old") + self.assertEqual(names(client.receive(old, 1)), ["old"]) + + client.receive(server.accept_connect(client.create_connect(2), 2), 2) + self.assertEqual(names(client.receive(old, 3)), [], "an earlier generation's frame was delivered again") + self.assertTrue(self.names_sequence(client.create_ack_for_received(old.sequence), old.sequence)) + self.assertEqual(names(client.receive(send(server, "fresh"), 4)), ["fresh"]) + + def test_a_duplicate_below_the_receive_window_is_acknowledged_by_name(self) -> None: + client, server = session(10), session(500) + handshake(client, server) + first = send(client, "first") + self.assertEqual(names(server.receive(first, 1)), ["first"]) + for _ in range(4_200): + server.receive(send(client, "x"), 1) + client.receive(server.create_ack(), 1) + self.assertEqual(names(server.receive(first, 2)), []) + self.assertEqual( + server.create_ack_for_received(first.sequence).ack, + first.sequence, + "a duplicate below the window was not acknowledged by name", + ) + + def test_a_delayed_connect_from_an_earlier_attempt_does_not_strand_the_client(self) -> None: + client, server = session(10), session(500) + earlier = client.create_connect(0) + retried = client.create_connect(300) + accept = server.accept_connect(retried, 301) + client.receive(accept, 302) + server.receive(client.create_ack_for_received(accept.sequence), 302) + self.assertTrue(client.connected) + a = send(client, "a") + self.assertEqual(names(server.receive(a, 303)), ["a"]) + client.receive(server.create_ack_for_received(a.sequence), 303) + + self.assertTrue(server.connect_repeats(earlier), "the delayed Connect starts nothing") + reply = server.accept_connect(earlier, 304) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + client.receive(reply, 305) + self.assertTrue(client.connected) + + b = send(client, "b") + self.assertEqual(names(server.receive(b, 306)), ["b"], "the delayed Connect reset the server") + client.receive(server.create_ack_for_received(b.sequence), 307) + self.assertNotIn(b.sequence, client.pending_reliable_sequences) + + def test_a_repeated_connect_refreshes_liveness_and_a_stale_one_does_not(self) -> None: + client, server = session(10), session(500) + earlier = client.create_connect(0) + current = client.create_connect(1) + server.accept_connect(current, 0) + + server.accept_connect(current, 900) + self.assertFalse(server.check_timeout(1_000, 500), "a repeated Connect did not refresh liveness") + + server.accept_connect(earlier, 1_400) + self.assertTrue(server.check_timeout(1_600, 500), "a stale Connect refreshed liveness") + + def test_stale_connects_are_recognised_by_serial_arithmetic(self) -> None: + server = session(500) + server.accept_connect(session(3).create_connect(0), 0) + + def connect(sequence: int): + return session(sequence).create_connect(0) + + self.assertTrue(server.connect_repeats(connect(2**32 - 2)), "just before the wrap") + self.assertFalse(server.connect_repeats(connect(4)), "just after") + self.assertFalse(server.connect_repeats(connect(3 + 4_096)), "far after") + self.assertTrue(server.connect_repeats(connect(2**32 - 4_092)), "the window's edge") + self.assertFalse(server.connect_repeats(connect(2**32 - 4_093)), "past the window") + + def test_a_restarted_client_whose_first_sequence_is_stale_connects_after_redrawing(self) -> None: + server, old = session(500), session(1_000) + old_accept = server.accept_connect(old.create_connect(0), 0) + old.receive(old_accept, 0) + + restarted = session(900) + first = restarted.create_connect(0, b"join") + # The old client has not acknowledged its Accept yet, so that is the + # reply: it names the old Connect, not this one. + early = server.accept_connect(first, 1) + self.assertEqual(early.packet_type, CultNetRudpPacketType.ACCEPT) + restarted.receive(early, 1) + self.assertFalse(restarted.connected, "an Accept for another Connect connected the client") + server.receive(old.create_ack_for_received(old_accept.sequence), 1) + reply = server.accept_connect(first, 2) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + restarted.receive(reply, 2) + self.assertFalse(restarted.connected) + + retransmitted = restarted.due_resends(1_000) + self.assertEqual(len(retransmitted), 1) + self.assertEqual(retransmitted[0].sequence, first.sequence, "the attempt is still young") + + fresh = restarted.due_resends(3_500) + self.assertEqual(len(fresh), 1) + self.assertEqual(fresh[0].packet_type, CultNetRudpPacketType.CONNECT) + self.assertNotEqual(fresh[0].sequence, first.sequence, "the same Connect was retransmitted for ever") + self.assertEqual(fresh[0].payload, b"join", "the fresh attempt lost the Connect payload") + + accept = server.accept_connect(fresh[0], 3_500) + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + restarted.receive(accept, 3_500) + self.assertTrue(restarted.connected) + self.assertEqual(names(server.receive(send(restarted, "hello"), 3_501)), ["hello"]) + + def test_an_answered_connect_is_never_replaced_by_the_attempt_timeout(self) -> None: + client, server = session(10), session(500) + connect, _ = handshake(client, server) + resent = client.due_resends(60_000) + self.assertTrue( + all(p.packet_type != CultNetRudpPacketType.CONNECT or p.sequence == connect.sequence for p in resent), + "a connected client started a new attempt", + ) + self.assertTrue(client.connected) + + def test_an_accepted_peer_that_goes_silent_times_out(self) -> None: + server = session(500) + server.accept_connect(session(10).create_connect(0), 0) + self.assertTrue(server.check_timeout(100_000, 1_000)) + + + # Ack Cut 1d, batch 2: the connect-attempt timeout. + + def test_an_ack_naming_the_pending_connect_does_not_retire_it(self) -> None: + server, old = session(500), session(1) + old_accept = server.accept_connect(old.create_connect(0), 0) + old.receive(old_accept, 0) + server.receive(old.create_ack_for_received(old_accept.sequence), 0) + + restarted = session(1) + first = restarted.create_connect(0, b"join") + reply = server.accept_connect(first, 1) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + self.assertTrue(self.names_sequence(reply, first.sequence), "the Ack must name the pending Connect for this test to bite") + restarted.receive(reply, 1) + self.assertFalse(restarted.connected) + self.assertEqual(restarted.pending_reliable_sequences, (first.sequence,), "an Ack retired the Connect") + + fresh = restarted.due_resends(3_000) + self.assertEqual(len(fresh), 1, "the client waits for ever with nothing to resend") + self.assertEqual(fresh[0].payload, b"join") + accept = server.accept_connect(fresh[0], 3_000) + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + restarted.receive(accept, 3_000) + self.assertTrue(restarted.connected) + + def test_a_late_copy_of_an_abandoned_connect_is_stale_once_its_replacement_is_accepted(self) -> None: + server, client = session(500), session(10) + abandoned = client.create_connect(0) + fresh = client.due_resends(3_000) + self.assertEqual(len(fresh), 1) + self.assertEqual(fresh[0].sequence, abandoned.sequence + 4_095) + accept = server.accept_connect(fresh[0], 3_001) + client.receive(accept, 3_002) + server.receive(client.create_ack_for_received(accept.sequence), 3_002) + self.assertTrue(client.connected) + a = send(client, "a") + self.assertEqual(names(server.receive(a, 3_003)), ["a"]) + client.receive(server.create_ack_for_received(a.sequence), 3_003) + + self.assertTrue(server.connect_repeats(abandoned), "the late copy would restart the server") + reply = server.accept_connect(abandoned, 3_004) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + client.receive(reply, 3_005) + self.assertTrue(client.connected) + self.assertEqual(names(server.receive(send(client, "b"), 3_006)), ["b"]) + + def test_a_server_sitting_at_the_jump_is_left_by_the_next_attempt(self) -> None: + server, old = session(500), session(5_095) + old_accept = server.accept_connect(old.create_connect(0), 0) + old.receive(old_accept, 0) + server.receive(old.create_ack_for_received(old_accept.sequence), 0) + + restarted = session(1_000) + first = restarted.create_connect(0, b"join") + restarted.receive(server.accept_connect(first, 1), 1) + self.assertFalse(restarted.connected) + + second = restarted.due_resends(3_000)[0] + self.assertEqual(second.sequence, 5_095, "the jump lands on the server's generation") + reply = server.accept_connect(second, 3_001) + self.assertEqual(reply.packet_type, CultNetRudpPacketType.ACK) + restarted.receive(reply, 3_001) + self.assertFalse(restarted.connected) + + third = restarted.due_resends(6_000)[0] + self.assertEqual(third.sequence, 5_095 + 4_095) + accept = server.accept_connect(third, 6_001) + self.assertEqual(accept.packet_type, CultNetRudpPacketType.ACCEPT) + restarted.receive(accept, 6_001) + self.assertTrue(restarted.connected) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-py/tests/test_rudp_ordered_delivery.py b/packages/cultnet-py/tests/test_rudp_ordered_delivery.py new file mode 100644 index 00000000..0417aeb0 --- /dev/null +++ b/packages/cultnet-py/tests/test_rudp_ordered_delivery.py @@ -0,0 +1,115 @@ +"""Ordered delivery has one owner: the contiguous received-through watermark. + +A frame is held for exactly one reason, a reliable sequence below it has not +arrived, and it is delivered in the call that fills the last such gap, +whichever channel the gap belonged to. +""" +from __future__ import annotations + +import unittest + +from cultnet_py.transport import ( + CultNetRudpPacket, + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, +) + + +def handshake() -> tuple[CultNetRudpSession, CultNetRudpSession]: + client = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=410, initial_sequence=1)) + server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=410, initial_sequence=500)) + accept = server.accept_connect(client.create_connect(0), 0) + client.receive(accept, 0) + return client, server + + +def send(session: CultNetRudpSession, channel: str, payload: str, ordered: bool) -> CultNetRudpPacket: + return session.send(channel, payload.encode(), CultNetRudpSendOptions(reliable=True, ordered=ordered)) + + +def names(result) -> list[str]: + return [frame.payload.decode() for frame in result.delivered] + + +class CultNetRudpOrderedDeliveryTests(unittest.TestCase): + def test_ordered_frame_waits_for_a_gap_filled_by_another_channel(self) -> None: + sender, receiver = handshake() + o1 = send(sender, "schema", "o1", True) + u = send(sender, "rel", "u", False) + o2 = send(sender, "schema", "o2", True) + o3 = send(sender, "schema", "o3", True) + o4 = send(sender, "schema", "o4", True) + + self.assertEqual(names(receiver.receive(o1, 1)), ["o1"]) + self.assertEqual(names(receiver.receive(o2, 2)), [], "u is missing") + self.assertEqual(names(receiver.receive(o3, 3)), [], "u is missing") + self.assertEqual(names(receiver.receive(o4, 4)), [], "u is missing") + self.assertEqual(names(receiver.receive(u, 5)), ["u", "o2", "o3", "o4"]) + + def test_two_ordered_channels_release_each_other_in_sequence_order(self) -> None: + sender, receiver = handshake() + a1 = send(sender, "schema", "A1", True) + b1 = send(sender, "other", "B1", True) + a2 = send(sender, "schema", "A2", True) + a3 = send(sender, "schema", "A3", True) + + delivered = names(receiver.receive(a1, 1)) + names(receiver.receive(a2, 2)) + names(receiver.receive(a3, 3)) + self.assertEqual(delivered, ["A1"], "B1 is missing, so A2 and A3 wait") + self.assertEqual(names(receiver.receive(b1, 4)), ["B1", "A2", "A3"]) + + def test_channel_first_used_after_other_traffic_loses_nothing(self) -> None: + sender, receiver = handshake() + c1 = send(sender, "late", "C1", True) + c2 = send(sender, "late", "C2", True) + x = send(sender, "schema", "X", True) + + self.assertEqual(names(receiver.receive(x, 1)), [], "C1 is missing") + self.assertEqual(names(receiver.receive(c2, 2)), [], "C1 is missing") + self.assertEqual(names(receiver.receive(c1, 3)), ["C1", "C2", "X"]) + + def test_accept_seeds_the_watermark_of_the_connecting_side(self) -> None: + client, server = handshake() + s1 = send(server, "schema", "s1", True) + s2 = send(server, "schema", "s2", True) + + self.assertEqual(names(client.receive(s2, 1)), []) + self.assertEqual(names(client.receive(s1, 2)), ["s1", "s2"]) + + def test_connect_seeds_the_watermark_of_the_accepting_side(self) -> None: + client, server = handshake() + first = send(client, "schema", "first", True) + second = send(client, "schema", "second", True) + + self.assertEqual(names(server.receive(second, 1)), []) + self.assertEqual(names(server.receive(first, 2)), ["first", "second"]) + + def test_duplicate_of_a_held_frame_is_not_delivered_twice(self) -> None: + sender, receiver = handshake() + s1 = send(sender, "schema", "s1", True) + s2 = send(sender, "schema", "s2", True) + + self.assertEqual(names(receiver.receive(s2, 1)), []) + self.assertEqual(names(receiver.receive(s2, 2)), []) + self.assertEqual(names(receiver.receive(s1, 3)), ["s1", "s2"]) + self.assertEqual(names(receiver.receive(s2, 4)), []) + + def test_reset_peer_state_forgets_held_frames_and_reseeds_the_watermark_from_the_next_handshake(self) -> None: + sender, receiver = handshake() + send(sender, "schema", "s1", True) + s2 = send(sender, "schema", "s2", True) + self.assertEqual(names(receiver.receive(s2, 1)), []) + + receiver.reset_peer_state() + following = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=410, initial_sequence=100)) + following.receive(receiver.accept_connect(following.create_connect(0), 2), 2) + d1 = send(following, "schema", "d1", True) + d2 = send(following, "schema", "d2", True) + + # The held s2 died with the reset, and the watermark follows the new peer. + self.assertEqual(names(receiver.receive(d2, 3)), []) + self.assertEqual(names(receiver.receive(d1, 4)), ["d1", "d2"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-py/tests/test_rudp_session_ending.py b/packages/cultnet-py/tests/test_rudp_session_ending.py new file mode 100644 index 00000000..6e042b1c --- /dev/null +++ b/packages/cultnet-py/tests/test_rudp_session_ending.py @@ -0,0 +1,223 @@ +"""A write belongs to the session generation it was issued in. + +Every way a generation ends drops the writes it still owed, and a flush +belongs to the generation it started in. +""" +from __future__ import annotations + +import socket +import threading +import time +import unittest + +from cultnet_py.transport import ( + CultNetRudpPacketType, + CultNetRudpSendOptions, + CultNetRudpSession, + CultNetRudpSessionOptions, + CultNetRudpSocketMode, + CultNetRudpSocketTransportConnection, + CultNetRudpSocketTransportOptions, + decode_rudp_packet, + encode_rudp_packet, +) + +CONNECTION_ID = 0x10203070 + + +def bind_udp_socket() -> socket.socket: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 0)) + sock.settimeout(0.02) + return sock + + +def client_with_a_lost_write() -> tuple[CultNetRudpSession, CultNetRudpSession]: + client = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=1)) + server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=500)) + client.receive(server.accept_connect(client.create_connect(0), 0), 0) + client.send("schema", b"owed to the old session", CultNetRudpSendOptions(reliable=True, ordered=True)) + # A fragmented write larger than the send window leaves part of it queued. + client.send_many( + "schema", bytes(40 * 8), CultNetRudpSendOptions(reliable=True, ordered=True), max_fragment_bytes=8 + ) + assert client.outstanding_reliable_packet_count == 41 + assert client.queued_reliable_packet_count > 0 + return client, server + + +def drain(sock: socket.socket) -> list: + packets = [] + while True: + try: + wire, _ = sock.recvfrom(65535) + except TimeoutError: + return packets + packets.append(decode_rudp_packet(wire)) + + +def connected_client() -> tuple[CultNetRudpSocketTransportConnection, socket.socket, tuple]: + peer_socket = bind_udp_socket() + client = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-client", + socket=bind_udp_socket(), + mode=CultNetRudpSocketMode.CLIENT, + remote_addr=peer_socket.getsockname(), + connection_id=CONNECTION_ID, + ) + ) + client.connect(b"hello") + wire, client_addr = peer_socket.recvfrom(65535) + peer = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=500)) + accept = peer.accept_connect(decode_rudp_packet(wire), 0) + peer_socket.sendto(encode_rudp_packet(accept), client_addr) + client.receive_once() + assert client.connected + return client, peer_socket, client_addr + + +class CultNetRudpSessionEndingTests(unittest.TestCase): + def test_every_way_a_session_ends_drops_the_writes_it_owed(self) -> None: + endings = { + "peer Disconnect": lambda client, server: client.receive(server.create_disconnect(b"bye"), 1), + "local disconnect": lambda client, server: client.create_disconnect(b"bye"), + "timeout": lambda client, server: self.assertTrue(client.check_timeout(1_000, 10)), + "reset": lambda client, server: client.reset_peer_state(), + } + for name, end in endings.items(): + with self.subTest(ending=name): + client, server = client_with_a_lost_write() + end(client, server) + self.assertEqual(client.outstanding_reliable_packet_count, 0, "the write survived the end") + + next_server = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=900) + ) + next_server.accept_connect(client.create_connect(2_000), 2_000) + for resend in client.due_resends(60_000): + self.assertEqual( + next_server.receive(resend, 60_000).delivered, + (), + "the old session's write was delivered in the next session", + ) + + def test_a_flush_fails_after_a_local_disconnect_or_a_timeout(self) -> None: + client, peer_socket, _ = connected_client() + try: + client.send("schema", b"owed") + client.disconnect(b"bye") + with self.assertRaises(ConnectionError): + client.flush_reliable(0.2) + finally: + client.close() + peer_socket.close() + + client, peer_socket, _ = connected_client() + try: + client.send("schema", b"owed") + time.sleep(0.005) + self.assertTrue(client.check_timeout(1)) + with self.assertRaises(ConnectionError): + client.flush_reliable(0.2) + finally: + client.close() + peer_socket.close() + + def test_a_flush_that_started_before_the_end_fails_even_after_a_later_connect(self) -> None: + client, peer_socket, _ = connected_client() + outcome: list[BaseException | None] = [] + + def flush() -> None: + try: + client.flush_reliable(5.0) + outcome.append(None) + except BaseException as error: # noqa: BLE001 - the outcome is the assertion + outcome.append(error) + + try: + client.send("schema", b"owed") + worker = threading.Thread(target=flush) + worker.start() + time.sleep(0.05) + client.disconnect(b"bye") + client.connect(b"hello again") + worker.join(10) + self.assertEqual(len(outcome), 1) + self.assertIsInstance(outcome[0], ConnectionError) + finally: + client.close() + peer_socket.close() + + + def test_a_flush_started_after_a_reconnect_waits_on_the_new_session(self) -> None: + client, peer_socket, client_addr = connected_client() + try: + client.send("schema", b"owed to the old session") + client.disconnect(b"bye") + client.connect(b"hello again") + connect = [p for p in drain(peer_socket) if p.packet_type == CultNetRudpPacketType.CONNECT][-1] + accept = CultNetRudpSession( + CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=700) + ).accept_connect(connect, 0) + peer_socket.sendto(encode_rudp_packet(accept), client_addr) + client.receive_once() + + client.flush_reliable(0.5) + finally: + client.close() + peer_socket.close() + + def test_a_flush_started_after_the_server_accepts_a_new_connect_waits_on_the_new_session(self) -> None: + server_socket = bind_udp_socket() + peer_socket = bind_udp_socket() + server = CultNetRudpSocketTransportConnection( + CultNetRudpSocketTransportOptions( + runtime_id="python-rudp-server", + socket=server_socket, + mode=CultNetRudpSocketMode.SERVER, + connection_id=CONNECTION_ID, + ) + ) + to_server = lambda packet: peer_socket.sendto(encode_rudp_packet(packet), server_socket.getsockname()) + try: + first = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID)) + to_server(first.create_connect(0)) + server.receive_once() + to_server(first.create_disconnect()) + server.receive_once() + + following = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID)) + to_server(following.create_connect(0)) + server.receive_once() + accept = [p for p in drain(peer_socket) if p.packet_type == CultNetRudpPacketType.ACCEPT][-1] + following.receive(accept, 0) + to_server(following.create_ack()) + server.receive_once() + + server.flush_reliable(0.5) + finally: + server.close() + peer_socket.close() + + + def test_an_ending_does_not_forget_what_was_received_from_the_peer(self) -> None: + client = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=1)) + server = CultNetRudpSession(CultNetRudpSessionOptions(connection_id=CONNECTION_ID, initial_sequence=500)) + client.receive(server.accept_connect(client.create_connect(0), 0), 0) + ordered = CultNetRudpSendOptions(reliable=True, ordered=True) + s1 = client.send("schema", b"s1", ordered) + s2 = client.send("schema", b"s2", ordered) + self.assertEqual(server.receive(s2, 1).delivered, (), "s1 is missing, so s2 is held") + + self.assertTrue(server.check_timeout(1_000, 10)) + self.assertEqual( + [frame.payload for frame in server.receive(s1, 1_001).delivered], + [b"s1", b"s2"], + "the ending forgot the held frame", + ) + self.assertEqual(server.receive(s2, 1_002).delivered, (), "the ending forgot what was received") + + +if __name__ == "__main__": + unittest.main() diff --git a/packages/cultnet-rs/examples/cultnet_interop_peer.rs b/packages/cultnet-rs/examples/cultnet_interop_peer.rs index bd9ff774..a2a4847e 100644 --- a/packages/cultnet-rs/examples/cultnet_interop_peer.rs +++ b/packages/cultnet-rs/examples/cultnet_interop_peer.rs @@ -507,7 +507,7 @@ fn rudp_serve_once(options: &BTreeMap) -> Result<()> { let local_port = socket.local_addr()?.port(); let mut transport_options = CultNetRudpSocketTransportOptions::server("rust-rudp-interop", socket, 0x22446688); - transport_options.initial_sequence = 100; + transport_options.initial_sequence = Some(100); transport_options.resend_delay_ms = 25; transport_options.max_fragment_bytes = max_fragment_bytes; let mut transport = CultNetRudpSocketTransportConnection::new(transport_options)?; @@ -608,7 +608,7 @@ fn rudp_serve_message_once(options: &BTreeMap) -> Result<()> { let local_port = socket.local_addr()?.port(); let mut transport_options = CultNetRudpSocketTransportOptions::server("rust-rudp-message-interop", socket, 0x22446689); - transport_options.initial_sequence = 100; + transport_options.initial_sequence = Some(100); transport_options.resend_delay_ms = 25; let mut transport = CultNetRudpSocketTransportConnection::new(transport_options)?; @@ -967,7 +967,7 @@ fn run_rudp_server( let peer = peers.entry(remote_addr).or_insert_with(|| RudpPeerSession { session: CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: RUDP_INTEROP_CONNECTION_ID, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: RUDP_INTEROP_RESEND_DELAY_MS, max_pending_reliable_packets: None, }), diff --git a/packages/cultnet-rs/src/cultmesh.rs b/packages/cultnet-rs/src/cultmesh.rs index 14998396..900f4744 100644 --- a/packages/cultnet-rs/src/cultmesh.rs +++ b/packages/cultnet-rs/src/cultmesh.rs @@ -243,7 +243,7 @@ pub struct CultMeshRudpSocketOptions { pub bind_host: String, pub bind_port: u16, pub read_timeout: Option, - pub initial_sequence: u32, + pub initial_sequence: Option, pub resend_delay_ms: u64, pub max_payload_bytes: Option, pub max_fragment_bytes: Option, @@ -256,7 +256,7 @@ impl Default for CultMeshRudpSocketOptions { bind_host: "127.0.0.1".to_string(), bind_port: 0, read_timeout: Some(Duration::from_millis(20)), - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, max_payload_bytes: None, max_fragment_bytes: None, diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index b51abe87..3ee6af66 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -8,7 +8,6 @@ use std::net::SocketAddr; use std::net::UdpSocket; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use uuid::Uuid; use crate::CultNetMessage; use crate::CultNetReconnectController; @@ -46,6 +45,12 @@ const RUDP_MAX_TRACKED_CHANNELS: usize = 64; /// ahead of the current highest would otherwise advance the receive state past /// every legitimate sequence still in flight and starve the session. const RUDP_RECEIVE_AHEAD_WINDOW: u32 = 1_024; +/// How long a client keeps retransmitting a Connect nobody answered before it +/// abandons that attempt for a fresh one. A Connect the server answers with an +/// Ack, never an Accept, is one the server judged stale (see +/// `accept_connect`); retransmitting the same sequence would never change its +/// mind. +const RUDP_CONNECT_ATTEMPT_MS: u64 = 3_000; const RUDP_MAX_ORDERED_BUFFERED_FRAMES: usize = 1_024; const RUDP_MAX_ORDERED_BUFFERED_BYTES: usize = 4 * 1024 * 1024; @@ -98,7 +103,10 @@ pub struct CultNetRudpReceiveResult { #[derive(Clone, Debug, PartialEq, Eq)] pub struct CultNetRudpSessionOptions { pub connection_id: u32, - pub initial_sequence: u32, + /// The first sequence this session issues. Unset, the session draws its + /// own at random: every session is a new identity to its peer, and an + /// options object reused for several must not give them one sequence. + pub initial_sequence: Option, pub resend_delay_ms: u64, pub max_pending_reliable_packets: Option, } @@ -107,13 +115,41 @@ impl Default for CultNetRudpSessionOptions { fn default() -> Self { Self { connection_id: 0, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, max_pending_reliable_packets: None, } } } +/// A sequence drawn at random from [1, 2^31). The Connect's sequence is what +/// tells a peer whether a Connect repeats one it already accepted or starts a +/// new session, so two sessions must not share one by default. At least 2^31 +/// sequences remain before the space is exhausted. +fn draw_sequence() -> u32 { + rand::random_range(1..(1_u32 << 31)) +} + +/// Whether `sequence` is at or before `mark` in serial order, within the +/// receive window. The compare is modular, so it holds across the wrap of the +/// 32-bit space; a sequence further back than the window is the duplicate +/// test's below-window clause, and one ahead of the mark is never before it. +fn at_or_before(sequence: u32, mark: u32) -> bool { + mark.wrapping_sub(sequence) < RUDP_RECEIVED_SEQUENCE_WINDOW as u32 +} + +/// Whether the packet's ack field, or its mask, names `sequence`. +fn packet_acknowledges(packet: &CultNetRudpPacket, sequence: u32) -> bool { + if packet.ack == sequence { + return true; + } + (0..32).any(|bit| { + packet.ack_mask & (1_u32 << bit) != 0 + && packet.ack > bit + && packet.ack - bit - 1 == sequence + }) +} + #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct CultNetRudpSendOptions { pub reliable: bool, @@ -128,13 +164,13 @@ pub struct CultNetRudpSendOptions { } /// Identifies every transport packet belonging to one non-expiring reliable -/// send. Receipts are local to the session that issued them: a peer reset or -/// a Disconnect received from the peer invalidates them for good. A local -/// `disconnect()` or a timeout does not; the writes it leaves owed carry into -/// the next session. +/// send. A write belongs to the session generation it was issued in, and every +/// way a generation can end (a Disconnect from the peer, a local `disconnect()`, +/// a timeout, a refused packet) invalidates the receipts it issued for good. #[derive(Clone, Debug, PartialEq, Eq)] pub struct CultNetRudpReliableSendReceipt { - session_scope: Uuid, + session_id: u64, + generation: u64, sequences: Vec, } @@ -158,12 +194,6 @@ struct PendingReliablePacket { expires_at_ms: Option, } -#[derive(Clone, Debug, PartialEq, Eq)] -struct PendingOrderedFrame { - frame: CultNetRudpDeliveredFrame, - next_sequence: u32, -} - #[derive(Clone, Debug, PartialEq, Eq)] struct FragmentBuffer { channel_id: String, @@ -178,7 +208,28 @@ struct FragmentBuffer { } pub struct CultNetRudpSession { - session_scope: Uuid, + /// Random per session. A receipt carries its session's id, so one asked of + /// another session reads `Invalidated` instead of borrowing that session's + /// generation numbers. + session_id: u64, + /// Advances every time a generation ends. Everything issued in a generation + /// (writes, receipts, flushes) belongs to it and dies with it. + generation: u64, + /// True from the end of a generation until the next Connect or Accept begins + /// one. A flush started in that interval has no live generation to wait on. + ended: bool, + /// The sequence of the Connect that started the current generation: sent + /// by this side, or accepted from the peer. A Connect repeats exactly when + /// the session is connected and the Connect carries this sequence. + connect_sequence: Option, + /// A Connect this side sent is unanswered. Only then is an Accept honoured. + awaiting_accept: bool, + /// The payload of the Connect this side sent, kept so a fresh attempt can + /// always be built whatever became of the pending packet. + connect_payload: Vec, + /// When the unanswered Connect first went out; `due_resends` abandons it + /// for a fresh attempt once `RUDP_CONNECT_ATTEMPT_MS` has passed. + connect_started_at_ms: u64, connection_id: u32, resend_delay_ms: u64, max_pending_reliable_packets: Option, @@ -195,8 +246,15 @@ pub struct CultNetRudpSession { pending_reliable: BTreeMap, queued_reliable: VecDeque<(CultNetRudpPacket, Option)>, reliable_packets_expired: u64, - ordered_next_sequence_by_channel: BTreeMap, - ordered_buffers: BTreeMap>, + /// Every reliable sequence up to and including this one has been received + /// since the peer state was last reset. Only the handshake seeds it: the + /// peer's Connect on the accepting side, the peer's Accept on the + /// connecting side. Reliable data that arrives before that is refused. + received_through: Option, + /// Ordered frames received but not yet deliverable, keyed by first + /// sequence. A frame is held for exactly one reason: a reliable sequence + /// below it has not arrived. + ordered_held: BTreeMap, fragment_buffers: BTreeMap<(String, u16), FragmentBuffer>, fragment_touches: u64, fragment_sets_evicted: u64, @@ -205,13 +263,19 @@ pub struct CultNetRudpSession { impl CultNetRudpSession { pub fn new(options: CultNetRudpSessionOptions) -> Self { Self { - session_scope: Uuid::new_v4(), + session_id: rand::random(), + generation: 0, + ended: false, + connect_sequence: None, + awaiting_accept: false, + connect_payload: Vec::new(), + connect_started_at_ms: 0, connection_id: options.connection_id, resend_delay_ms: options.resend_delay_ms, max_pending_reliable_packets: options.max_pending_reliable_packets, max_payload_bytes: None, max_pending_fragment_sets: 64, - next_sequence: options.initial_sequence, + next_sequence: options.initial_sequence.unwrap_or_else(draw_sequence), next_sequenced_by_channel: BTreeMap::new(), next_fragment_id: 1, connected: false, @@ -222,8 +286,8 @@ impl CultNetRudpSession { pending_reliable: BTreeMap::new(), queued_reliable: VecDeque::new(), reliable_packets_expired: 0, - ordered_next_sequence_by_channel: BTreeMap::new(), - ordered_buffers: BTreeMap::new(), + received_through: None, + ordered_held: BTreeMap::new(), fragment_buffers: BTreeMap::new(), fragment_touches: 0, fragment_sets_evicted: 0, @@ -244,6 +308,7 @@ impl CultNetRudpSession { pub fn assume_connected(&mut self, now_ms: u64) { self.connected = true; + self.ended = false; self.last_received_at_ms = Some(now_ms); } @@ -282,15 +347,35 @@ impl CultNetRudpSession { Some(pending.packet.clone()) } - fn session_scope(&self) -> Uuid { - self.session_scope + fn generation(&self) -> u64 { + self.generation + } + + fn ended(&self) -> bool { + self.ended + } + + /// The one way a session generation ends. The session stops being + /// connected and what it still owed the peer dies with it: a write not yet + /// acknowledged is dropped, so no later session retransmits it or credits an + /// ack to it, and its receipt can never become Acknowledged. What was + /// learned from the peer is not touched: the peer may not know the session + /// ended, and forgetting what it sent would let its retransmits be + /// delivered twice. + fn end_session(&mut self) { + self.connected = false; + self.ended = true; + self.awaiting_accept = false; + self.generation += 1; + self.pending_reliable.clear(); + self.queued_reliable.clear(); } fn reliable_send_status( &self, receipt: &CultNetRudpReliableSendReceipt, ) -> CultNetRudpReliableSendStatus { - if receipt.session_scope != self.session_scope { + if receipt.session_id != self.session_id || receipt.generation != self.generation { return CultNetRudpReliableSendStatus::Invalidated; } if receipt @@ -314,7 +399,8 @@ impl CultNetRudpSession { )); } Ok(CultNetRudpReliableSendReceipt { - session_scope: self.session_scope, + session_id: self.session_id, + generation: self.generation, sequences: packets.iter().map(|packet| packet.sequence).collect(), }) } @@ -344,17 +430,14 @@ impl CultNetRudpSession { /// ids already issued stay issued: a peer that still remembers them must /// never see one reused. pub fn reset_peer_state(&mut self) { - self.session_scope = Uuid::new_v4(); + self.end_session(); self.next_sequenced_by_channel.clear(); - self.connected = false; self.last_received_at_ms = None; self.highest_received_sequence = None; self.received_sequences.clear(); self.latest_sequenced_by_channel.clear(); - self.pending_reliable.clear(); - self.queued_reliable.clear(); - self.ordered_next_sequence_by_channel.clear(); - self.ordered_buffers.clear(); + self.received_through = None; + self.ordered_held.clear(); self.fragment_buffers.clear(); } @@ -369,6 +452,12 @@ impl CultNetRudpSession { } pub fn create_connect(&mut self, now_ms: u64, payload: Vec) -> Result { + // A session that has had a peer starts a new generation: nothing it + // learned from that peer describes the one this Connect reaches, and + // what it still owed that peer no longer takes room in the queue. + if self.connect_sequence.is_some() { + self.reset_peer_state(); + } self.ensure_reliable_capacity(1)?; let packet = self.create_packet( CultNetRudpPacketType::Connect, @@ -378,10 +467,49 @@ impl CultNetRudpSession { true, false, ); + self.ended = false; + self.connect_sequence = Some(packet.sequence); + self.awaiting_accept = true; + self.connect_started_at_ms = now_ms; + self.connect_payload = packet.payload.clone(); self.track_reliable(packet.clone(), now_ms, None); Ok(packet) } + /// Whether `packet` is a Connect this session's current generation already + /// owns: a retransmit of the Connect that started it (the session is + /// connected and the sequence is that Connect's), or a stale copy of an + /// earlier attempt by the same client (a sequence before it, within the + /// receive window). Servers that keep one session per peer ask this to tell + /// a Connect that starts a new session from one that does not; anything + /// else that reaches `accept_connect` starts a new generation. + pub fn connect_repeats(&self, packet: &CultNetRudpPacket) -> bool { + packet.packet_type == CultNetRudpPacketType::Connect + && self.connected + && self.connect_sequence.is_some_and(|connect| { + connect == packet.sequence || self.connect_is_stale(packet.sequence, connect) + }) + } + + /// A Connect that precedes the current generation's within the receive + /// window is the client's earlier attempt, delayed in the network: a + /// client that retried never sends a lower sequence again. Restarting on + /// it would strand the client, which honours only the Accept for its + /// newest Connect. The rule is TCP's answer to a delayed SYN (RFC 5961's + /// challenge ACK): keep the connection and answer with an Ack. A restarted + /// client whose random initial sequence lands in this window is answered + /// the same way and abandons the attempt for a fresh draw + /// (`RUDP_CONNECT_ATTEMPT_MS`). + fn connect_is_stale(&self, sequence: u32, current: u32) -> bool { + sequence != current && at_or_before(sequence, current) + } + + /// Answers a Connect. A repeat of the accepted one queues nothing, so a + /// Connect storm cannot grow the reliable queue: the reply is the Accept + /// still awaiting acknowledgement, or an Ack once it was acknowledged. A + /// stale copy of an earlier attempt gets the same reply and changes + /// nothing else: it is not evidence the peer is alive. Any other Connect + /// ends the current generation, forgets the peer and accepts a new one. pub fn accept_connect( &mut self, packet: &CultNetRudpPacket, @@ -396,10 +524,24 @@ impl CultNetRudpSession { )); } + if self.connect_repeats(packet) { + if self.connect_sequence == Some(packet.sequence) { + self.apply_acknowledgements(packet); + self.remember_received(packet.sequence); + self.last_received_at_ms = Some(now_ms); + } + return Ok(self + .pending_accept_for_resend(now_ms) + .unwrap_or_else(|| self.create_ack())); + } + + self.reset_peer_state(); self.ensure_reliable_capacity(1)?; - self.remember_received(packet.sequence); + self.seed_received(packet.sequence); self.last_received_at_ms = Some(now_ms); + self.connect_sequence = Some(packet.sequence); self.connected = true; + self.ended = false; let response = self.create_packet( CultNetRudpPacketType::Accept, "control", @@ -412,30 +554,6 @@ impl CultNetRudpSession { Ok(response) } - /// Answers a Connect from a peer this session has already accepted. The - /// packet's sequence is remembered and nothing is queued, so a Connect storm - /// cannot grow the reliable queue. The reply is the Accept still awaiting - /// acknowledgement, or an Ack once it was acknowledged. - pub fn answer_repeated_connect( - &mut self, - packet: &CultNetRudpPacket, - now_ms: u64, - ) -> Result { - self.require_connection(packet)?; - if packet.packet_type != CultNetRudpPacketType::Connect { - return Err(anyhow!( - "Expected RUDP connect packet, got {:?}", - packet.packet_type - )); - } - self.apply_acknowledgements(packet); - self.remember_received(packet.sequence); - self.last_received_at_ms = Some(now_ms); - Ok(self - .pending_accept_for_resend(now_ms) - .unwrap_or_else(|| self.create_ack())) - } - pub fn send( &mut self, channel_id: &str, @@ -535,17 +653,38 @@ impl CultNetRudpSession { now_ms: u64, ) -> Result { self.require_connection(packet)?; - self.apply_acknowledgements(packet); + // An Accept counts only while this side's Connect is unanswered and the + // Accept names it. A late or duplicate one, or one from an earlier + // generation, must not seed the watermark or revive an ended session. + let honours_accept = packet.packet_type == CultNetRudpPacketType::Accept + && self.awaiting_accept + && self + .connect_sequence + .is_some_and(|connect| packet_acknowledges(packet, connect)); + if packet.packet_type == CultNetRudpPacketType::Accept && !honours_accept { + return Ok(CultNetRudpReceiveResult { + delivered: Vec::new(), + ready_to_send: Vec::new(), + reply: None, + pong: false, + pong_payload: Vec::new(), + disconnected: false, + disconnect_reason: Vec::new(), + }); + } + // While this side's Connect awaits its Accept, only the Accept it + // honours retires it: an Ack that names the Connect (a server's reply + // to a repeat or a stale copy) says the server did not start a session. + if honours_accept || !self.awaiting_accept { + self.apply_acknowledgements(packet); + } self.purge_expired_reliable(now_ms); let ready_to_send = self.promote_queued_reliable(now_ms); self.last_received_at_ms = Some(now_ms); - let expected_sequence_if_uninitialized = self - .highest_received_sequence - .map(|sequence| sequence + 1) - .unwrap_or(packet.sequence); - if packet.packet_type == CultNetRudpPacketType::Accept { - self.remember_received(packet.sequence); + if honours_accept { + self.awaiting_accept = false; + self.seed_received(packet.sequence); self.connected = true; return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), @@ -596,13 +735,7 @@ impl CultNetRudpSession { } if packet.packet_type == CultNetRudpPacketType::Disconnect { - // The session is over. What it still owed the peer dies with it, - // and receipts it issued can never become Acknowledged in a later - // session. - self.connected = false; - self.session_scope = Uuid::new_v4(); - self.pending_reliable.clear(); - self.queued_reliable.clear(); + self.end_session(); return Ok(CultNetRudpReceiveResult { delivered: Vec::new(), ready_to_send, @@ -626,6 +759,21 @@ impl CultNetRudpSession { }); } + // Reliable data before the handshake has seeded the watermark has no + // place in the order: refuse it unremembered, so it is not acknowledged + // and the sender retransmits it once the handshake is done. + if packet.reliable && self.received_through.is_none() { + return Ok(CultNetRudpReceiveResult { + delivered: Vec::new(), + ready_to_send, + reply: None, + pong: false, + pong_payload: Vec::new(), + disconnected: false, + disconnect_reason: Vec::new(), + }); + } + let far_ahead = packet.reliable && self.highest_received_sequence.is_some_and(|highest| { packet.sequence > highest @@ -645,12 +793,7 @@ impl CultNetRudpSession { }); } - let duplicate = packet.reliable - && (self.received_sequences.contains(&packet.sequence) - || self.highest_received_sequence.is_some_and(|highest| { - packet.sequence < highest - && highest - packet.sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW as u32 - })); + let duplicate = packet.reliable && self.was_received(packet.sequence); if packet.reliable { self.remember_received(packet.sequence); } @@ -666,42 +809,36 @@ impl CultNetRudpSession { }); } - let Some((frame, ordered, next_sequence)) = self.reassemble(packet)? else { - return Ok(CultNetRudpReceiveResult { - delivered: Vec::new(), - ready_to_send, - reply: None, - pong: false, - pong_payload: Vec::new(), - disconnected: false, - disconnect_reason: Vec::new(), - }); - }; - let delivered = if ordered { - self.deliver_ordered(frame, next_sequence, expected_sequence_if_uninitialized)? - } else if packet.sequenced { - if !self - .latest_sequenced_by_channel - .contains_key(&frame.channel_id) - && self.latest_sequenced_by_channel.len() >= RUDP_MAX_TRACKED_CHANNELS - { - return Err(anyhow!("RUDP session has too many sequenced channels")); - } - let latest = self - .latest_sequenced_by_channel - .get(&frame.channel_id) - .copied(); - let newest_sequence = next_sequence.saturating_sub(1); - if latest.is_some_and(|latest| newest_sequence <= latest) { - Vec::new() + let mut delivered = Vec::new(); + if let Some((frame, ordered, next_sequence)) = self.reassemble(packet)? { + if ordered { + self.ordered_held.insert(frame.sequence, frame); + } else if packet.sequenced { + if !self + .latest_sequenced_by_channel + .contains_key(&frame.channel_id) + && self.latest_sequenced_by_channel.len() >= RUDP_MAX_TRACKED_CHANNELS + { + return Err(anyhow!("RUDP session has too many sequenced channels")); + } + let latest = self + .latest_sequenced_by_channel + .get(&frame.channel_id) + .copied(); + let newest_sequence = next_sequence.saturating_sub(1); + if !latest.is_some_and(|latest| newest_sequence <= latest) { + self.latest_sequenced_by_channel + .insert(frame.channel_id.clone(), newest_sequence); + delivered.push(frame); + } } else { - self.latest_sequenced_by_channel - .insert(frame.channel_id.clone(), newest_sequence); - vec![frame] + delivered.push(frame); } - } else { - vec![frame] - }; + } + // Any reliable packet may have advanced the watermark, a fragment or an + // unordered frame as much as an ordered one, so the drain runs after all + // of them. + delivered.extend(self.drain_ordered()?); Ok(CultNetRudpReceiveResult { delivered, ready_to_send, @@ -750,9 +887,14 @@ impl CultNetRudpSession { } } + /// The acknowledgement for a reliable packet `receive` just handled. What + /// `receive` refused is not acknowledged by name (the ack carries only what + /// was received), so its sender retransmits it. pub fn create_ack_for_received(&mut self, sequence: u32) -> CultNetRudpPacket { let (ack, _) = self.ack_state(); - if ack >= sequence && ack - sequence <= 32 { + if !self.was_received(sequence) { + self.create_ack() + } else if ack >= sequence && ack - sequence <= 32 { self.create_ack() } else { self.create_ack_for(sequence) @@ -771,7 +913,7 @@ impl CultNetRudpSession { } pub fn create_disconnect(&mut self, reason: Vec) -> CultNetRudpPacket { - self.connected = false; + self.end_session(); self.create_packet( CultNetRudpPacketType::Disconnect, "control", @@ -792,11 +934,14 @@ impl CultNetRudpSession { if now_ms.saturating_sub(last_received_at_ms) <= timeout_ms { return false; } - self.connected = false; + self.end_session(); true } pub fn due_resends(&mut self, now_ms: u64) -> Vec { + if let Some(fresh) = self.abandon_unanswered_connect(now_ms) { + return vec![fresh]; + } let mut due = Vec::new(); for pending in self.pending_reliable.values_mut() { if now_ms.saturating_sub(pending.last_sent_at_ms) >= self.resend_delay_ms { @@ -808,6 +953,30 @@ impl CultNetRudpSession { due } + /// A Connect unanswered for `RUDP_CONNECT_ATTEMPT_MS` is replaced, not + /// retransmitted further, by a Connect with the same payload and the + /// abandoned sequence plus the receive window less one. Nothing was ever + /// sent in an unanswered generation, so no sequence issued so far is owed. + /// The jump keeps a late copy of the abandoned Connect inside the new one's + /// stale window, so a server that took the new Connect answers the copy + /// with an Ack instead of restarting; and it leaves the stale window of + /// whatever generation the server holds, unless that generation sits + /// exactly at the jump, and then the next attempt leaves it. The first + /// Connect of a session is the only one drawn at random. + fn abandon_unanswered_connect(&mut self, now_ms: u64) -> Option { + if !self.awaiting_accept + || now_ms.saturating_sub(self.connect_started_at_ms) < RUDP_CONNECT_ATTEMPT_MS + { + return None; + } + let abandoned = self.connect_sequence?; + self.next_sequence = abandoned + .saturating_add(RUDP_RECEIVED_SEQUENCE_WINDOW as u32 - 1) + .min(u32::MAX - 1); + let payload = self.connect_payload.clone(); + self.create_connect(now_ms, payload).ok() + } + fn create_packet( &mut self, packet_type: CultNetRudpPacketType, @@ -972,8 +1141,24 @@ impl CultNetRudpSession { } } + /// The handshake's one act on the watermark: the peer's Connect or Accept + /// is the first sequence of the session, whatever else it has sent. + fn seed_received(&mut self, sequence: u32) { + self.received_through = Some(sequence); + self.remember_received(sequence); + } + fn remember_received(&mut self, sequence: u32) { self.received_sequences.insert(sequence); + if let Some(mut through) = self.received_through { + while let Some(next) = through.checked_add(1) { + if !self.received_sequences.contains(&next) { + break; + } + through = next; + } + self.received_through = Some(through); + } if self .highest_received_sequence .is_none_or(|highest| sequence > highest) @@ -989,6 +1174,22 @@ impl CultNetRudpSession { } } + /// The one duplicate test: true for a sequence in the window that was + /// received, for one below the window, and for one at or before the + /// watermark. The watermark starts at the handshake's seed, so a frame the + /// peer sent in an earlier generation (every sequence it issued is below + /// the Connect that began this one) is a duplicate of something already + /// delivered and is acknowledged, never delivered again. + fn was_received(&self, sequence: u32) -> bool { + self.received_sequences.contains(&sequence) + || self.highest_received_sequence.is_some_and(|highest| { + sequence < highest && highest - sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW as u32 + }) + || self + .received_through + .is_some_and(|through| at_or_before(sequence, through)) + } + fn ack_state(&self) -> (u32, u32) { let ack = self.highest_received_sequence.unwrap_or(0); let mut ack_mask = 0_u32; @@ -1126,126 +1327,39 @@ impl CultNetRudpSession { Ok(()) } - fn ordered_buffered_frames(&self) -> usize { - self.ordered_buffers.values().map(BTreeMap::len).sum() - } - - fn ordered_buffered_bytes(&self) -> usize { - self.ordered_buffers - .values() - .flat_map(BTreeMap::values) - .map(|pending| pending.frame.payload.len()) - .sum() - } - - fn deliver_ordered( - &mut self, - frame: CultNetRudpDeliveredFrame, - next_sequence_after_frame: u32, - expected_sequence_if_uninitialized: u32, - ) -> Result> { - let channel_id = frame.channel_id.clone(); - if !self.ordered_next_sequence_by_channel.contains_key(&channel_id) - && self.ordered_next_sequence_by_channel.len() >= RUDP_MAX_TRACKED_CHANNELS + /// Delivers, in sequence order, every held ordered frame whose first + /// sequence is at most one past the watermark, then bounds what is still + /// held. Ordered delivery has one owner: the watermark. A frame this call + /// does not deliver waits for the sequence below it, and nothing else + /// releases it. + fn drain_ordered(&mut self) -> Result> { + let Some(through) = self.received_through else { + return Ok(Vec::new()); + }; + let still_held = match through.checked_add(1) { + Some(next) => self.ordered_held.split_off(&next.saturating_add(1)), + None => BTreeMap::new(), + }; + let deliverable = std::mem::replace(&mut self.ordered_held, still_held); + if self.ordered_held.len() > RUDP_MAX_ORDERED_BUFFERED_FRAMES + || self + .ordered_held + .values() + .map(|frame| frame.payload.len()) + .sum::() + > RUDP_MAX_ORDERED_BUFFERED_BYTES { + return Err(anyhow!("RUDP ordered hold buffer is full")); + } + let held_channels: BTreeSet<&str> = self + .ordered_held + .values() + .map(|frame| frame.channel_id.as_str()) + .collect(); + if held_channels.len() > RUDP_MAX_TRACKED_CHANNELS { return Err(anyhow!("RUDP session has too many ordered channels")); } - let mut next = if let Some(next) = self - .ordered_next_sequence_by_channel - .get(&channel_id) - .copied() - { - next - } else { - self.ordered_next_sequence_by_channel.insert( - channel_id.clone(), - expected_sequence_if_uninitialized.min(frame.sequence), - ); - expected_sequence_if_uninitialized.min(frame.sequence) - }; - - while frame.sequence > next - && self.received_sequences.contains(&next) - && !self - .ordered_buffers - .get(&channel_id) - .is_some_and(|buffer| buffer.contains_key(&next)) - { - next = next.saturating_add(1); - self.ordered_next_sequence_by_channel - .insert(channel_id.clone(), next); - } - - if frame.sequence < next { - return Ok(Vec::new()); - } - - if frame.sequence > next { - if self.ordered_buffered_frames() >= RUDP_MAX_ORDERED_BUFFERED_FRAMES - || self.ordered_buffered_bytes() + frame.payload.len() - > RUDP_MAX_ORDERED_BUFFERED_BYTES - { - return Err(anyhow!("RUDP ordered hold buffer is full")); - } - self.ordered_buffers.entry(channel_id).or_default().insert( - frame.sequence, - PendingOrderedFrame { - frame, - next_sequence: next_sequence_after_frame, - }, - ); - return Ok(Vec::new()); - } - - self.ordered_next_sequence_by_channel - .insert(channel_id.clone(), next_sequence_after_frame); - let mut delivered = vec![frame]; - delivered.extend(self.drain_ordered(&channel_id)); - Ok(delivered) - } - - fn drain_ordered(&mut self, channel_id: &str) -> Vec { - let mut delivered = Vec::new(); - loop { - let Some(next) = self - .ordered_next_sequence_by_channel - .get(channel_id) - .copied() - else { - break; - }; - let Some(buffer) = self.ordered_buffers.get_mut(channel_id) else { - break; - }; - let Some(pending) = buffer.remove(&next) else { - break; - }; - delivered.push(pending.frame); - self.ordered_next_sequence_by_channel - .insert(channel_id.to_string(), pending.next_sequence); - self.skip_received_non_channel_sequences(channel_id); - } - delivered - } - - fn skip_received_non_channel_sequences(&mut self, channel_id: &str) { - let Some(mut next) = self - .ordered_next_sequence_by_channel - .get(channel_id) - .copied() - else { - return; - }; - while self.received_sequences.contains(&next) - && !self - .ordered_buffers - .get(channel_id) - .is_some_and(|buffer| buffer.contains_key(&next)) - { - next = next.saturating_add(1); - self.ordered_next_sequence_by_channel - .insert(channel_id.to_string(), next); - } + Ok(deliverable.into_values().collect()) } fn allocate_fragment_id(&mut self) -> u16 { @@ -1283,7 +1397,7 @@ pub struct CultNetRudpSocketTransportOptions { pub mode: CultNetRudpSocketMode, pub remote_addr: Option, pub connection_id: u32, - pub initial_sequence: u32, + pub initial_sequence: Option, pub resend_delay_ms: u64, pub transport_id: Option, pub max_payload_bytes: Option, @@ -1310,7 +1424,7 @@ impl CultNetRudpSocketTransportOptions { mode: CultNetRudpSocketMode::Client, remote_addr: Some(remote_addr), connection_id, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, transport_id: None, max_payload_bytes: None, @@ -1329,7 +1443,7 @@ impl CultNetRudpSocketTransportOptions { mode: CultNetRudpSocketMode::Server, remote_addr: None, connection_id, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, transport_id: None, max_payload_bytes: None, @@ -1357,7 +1471,7 @@ pub struct CultNetRudpServerHubOptions { pub runtime_id: String, pub socket: UdpSocket, pub connection_id: u32, - pub initial_sequence: u32, + pub initial_sequence: Option, pub resend_delay_ms: u64, pub transport_id: Option, pub max_payload_bytes: Option, @@ -1377,7 +1491,7 @@ impl CultNetRudpServerHubOptions { runtime_id: runtime_id.into(), socket, connection_id, - initial_sequence: 1, + initial_sequence: None, resend_delay_ms: 250, transport_id: None, max_payload_bytes: None, @@ -1432,7 +1546,7 @@ struct CultNetRudpServerPeer { pub struct CultNetRudpServerHub { socket: UdpSocket, connection_id: u32, - initial_sequence: u32, + initial_sequence: Option, resend_delay_ms: u64, max_pending_reliable_packets: Option, max_payload_bytes: usize, @@ -1638,9 +1752,9 @@ impl CultNetRudpServerHub { if packet.packet_type == CultNetRudpPacketType::Connect { if let Some(peer) = self.peers.get_mut(&remote_addr) - && peer.context.connect_payload == packet.payload + && peer.session.connect_repeats(&packet) { - let Ok(reply) = peer.session.answer_repeated_connect(&packet, now_ms()) else { + let Ok(reply) = peer.session.accept_connect(&packet, now_ms(), Vec::new()) else { self.end_refusing_peer(remote_addr)?; return Ok(true); }; @@ -1981,7 +2095,11 @@ impl CultNetRudpSocketTransportConnection { } pub fn check_timeout(&mut self, timeout_ms: u64) -> bool { - self.session.check_timeout(now_ms(), timeout_ms) + let timed_out = self.session.check_timeout(now_ms(), timeout_ms); + if timed_out { + self.disconnect_reason = Some(SESSION_TIMED_OUT_REASON.to_vec()); + } + timed_out } pub fn pending_reliable_packet_count(&self) -> usize { @@ -2000,18 +2118,19 @@ impl CultNetRudpSocketTransportConnection { self.socket.set_read_timeout(Some(poll_timeout))?; let deadline = Instant::now() + timeout; - // A flush belongs to the session it started in: a Connect that - // replaces the peer mid-wait forgets the writes being waited on. - let scope = self.session.session_scope(); + // A flush belongs to the generation it started in: whatever ends that + // generation, and whatever begins after it, the writes being waited on + // are gone. + let generation = self.session.generation(); let mut preserved_frames = VecDeque::new(); let result = (|| { loop { // An ended session forgot its unacknowledged writes; reporting // them flushed would be a lie. - if self.session.session_scope() != scope || self.disconnect_reason.is_some() { + if self.session.ended() || self.session.generation() != generation { return Err(anyhow!( "RUDP session ended before its reliable writes were acknowledged: {}", - String::from_utf8_lossy(self.disconnect_reason.as_deref().unwrap_or(b"replaced by a new Connect")) + String::from_utf8_lossy(self.disconnect_reason.as_deref().unwrap_or(b"no reason given")) )); } if self.outstanding_reliable_packet_count() == 0 { @@ -2086,12 +2205,17 @@ impl CultNetRudpSocketTransportConnection { self.stats.packets_dropped += 1; return Ok(true); } + // A Connect from another endpoint is a new client, whatever sequence it + // carries: the endpoint moves and the Connect starts a new generation. + // Only a Connect from the peer's own endpoint can be its repeat. + let mut connect_from_new_endpoint = false; if let Some(expected) = self.remote_addr { if expected != remote_addr { if self.mode == CultNetRudpSocketMode::Server && packet.packet_type == CultNetRudpPacketType::Connect { self.remote_addr = Some(remote_addr); + connect_from_new_endpoint = true; } else { self.stats.packets_dropped += 1; return Ok(true); @@ -2109,8 +2233,10 @@ impl CultNetRudpSocketTransportConnection { if self.mode == CultNetRudpSocketMode::Server && packet.packet_type == CultNetRudpPacketType::Connect { - self.session.reset_peer_state(); self.disconnect_reason = None; + if connect_from_new_endpoint { + self.session.reset_peer_state(); + } let Ok(accept) = self.session.accept_connect(&packet, now_ms(), Vec::new()) else { self.stats.packets_dropped += 1; return Ok(true); @@ -2278,14 +2404,15 @@ where /// Sent to a peer whose session refused one of its packets, and reported to /// the local caller as the disconnect reason. const RUDP_REFUSED_PACKET_REASON: &[u8] = b"session refused a packet"; +const SESSION_TIMED_OUT_REASON: &[u8] = b"session timed out"; /// A session cannot admit a Connect when its sequence space starts exhausted or /// its reliable queue holds nothing, so neither is a usable configuration. fn validate_rudp_session_limits( - initial_sequence: u32, + initial_sequence: Option, max_pending_reliable_packets: Option, ) -> Result<()> { - if initial_sequence == u32::MAX { + if initial_sequence == Some(u32::MAX) { return Err(anyhow!( "RUDP initial_sequence must leave room for a reliable packet" )); diff --git a/packages/cultnet-rs/tests/cultnet.rs b/packages/cultnet-rs/tests/cultnet.rs index 845ec846..46e6b2e2 100644 --- a/packages/cultnet-rs/tests/cultnet.rs +++ b/packages/cultnet-rs/tests/cultnet.rs @@ -552,13 +552,13 @@ fn rudp_reconnect_loop_consumes_shared_controller() -> Result<()> { fn rudp_session_handshake_acks_reliable_connect_and_accept_packets() -> Result<()> { let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 0x0a0b0c0d, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 50, ..CultNetRudpSessionOptions::default() }); let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 0x0a0b0c0d, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 50, ..CultNetRudpSessionOptions::default() }); @@ -589,52 +589,19 @@ fn rudp_session_handshake_acks_reliable_connect_and_accept_packets() -> Result<( fn rudp_session_computes_ack_masks_and_clears_pending_reliable_packets() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 7, - initial_sequence: 10, + initial_sequence: Some(10), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 7, - initial_sequence: 200, + initial_sequence: Some(200), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); - sender.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 7, - sequence: 1, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - receiver.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 7, - sequence: 2, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let options = CultNetRudpSendOptions { reliable: true, @@ -644,20 +611,20 @@ fn rudp_session_computes_ack_masks_and_clears_pending_reliable_packets() -> Resu let first = sender.send("schema", b"first".to_vec(), options.clone())?; let second = sender.send("schema", b"second".to_vec(), options.clone())?; let third = sender.send("schema", b"third".to_vec(), options)?; - assert_eq!(sender.pending_reliable_sequences(), vec![10, 11, 12]); + assert_eq!(sender.pending_reliable_sequences(), vec![11, 12, 13]); receiver.receive(&first, 0)?; receiver.receive(&third, 0)?; let ack_with_gap = receiver.create_ack(); - assert_eq!(ack_with_gap.ack, 12); - assert_eq!(ack_with_gap.ack_mask, 0b10 | (1 << 9)); + assert_eq!(ack_with_gap.ack, 13); + assert_eq!(ack_with_gap.ack_mask, 0b110); sender.receive(&ack_with_gap, 0)?; - assert_eq!(sender.pending_reliable_sequences(), vec![11]); + assert_eq!(sender.pending_reliable_sequences(), vec![12]); receiver.receive(&second, 0)?; let full_ack = receiver.create_ack(); - assert_eq!(full_ack.ack, 12); - assert_eq!(full_ack.ack_mask, 0b11 | (1 << 9)); + assert_eq!(full_ack.ack, 13); + assert_eq!(full_ack.ack_mask, 0b111); sender.receive(&full_ack, 0)?; assert!(sender.pending_reliable_sequences().is_empty()); Ok(()) @@ -667,28 +634,11 @@ fn rudp_session_computes_ack_masks_and_clears_pending_reliable_packets() -> Resu fn rudp_session_schedules_reliable_resends_until_acked() -> Result<()> { let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 99, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 99, - sequence: 50, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + session.assume_connected(0); let sent = session.send( "schema", b"payload".to_vec(), @@ -737,12 +687,12 @@ fn rudp_session_schedules_reliable_resends_until_acked() -> Result<()> { fn rudp_session_pings_and_detects_receive_timeout() -> Result<()> { let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 101, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 101, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = client.create_connect(0, b"join".to_vec())?; @@ -768,12 +718,12 @@ fn rudp_session_pings_and_detects_receive_timeout() -> Result<()> { fn rudp_lossy_packets_cannot_create_reliable_ordered_gaps() -> Result<()> { let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 198, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 198, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = client.create_connect(0, Vec::new())?; @@ -824,12 +774,12 @@ fn rudp_lossy_packets_cannot_create_reliable_ordered_gaps() -> Result<()> { fn rudp_unreliable_sequenced_delivery_is_scoped_to_its_channel() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 197, - initial_sequence: 50, + initial_sequence: Some(50), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 197, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -855,24 +805,7 @@ fn rudp_rejects_unreliable_ordered_delivery() -> Result<()> { connection_id: 196, ..CultNetRudpSessionOptions::default() }); - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 196, - sequence: 50, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + session.assume_connected(0); let error = session .send( "schema", @@ -895,28 +828,11 @@ fn rudp_rejects_unreliable_ordered_delivery() -> Result<()> { fn rudp_session_bounds_pending_reliable_packets_before_enqueue() -> Result<()> { let mut session = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 102, - initial_sequence: 1, + initial_sequence: Some(1), max_pending_reliable_packets: Some(2), ..CultNetRudpSessionOptions::default() }); - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 102, - sequence: 50, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + session.assume_connected(0); session.send( "schema", b"first".to_vec(), @@ -951,28 +867,11 @@ fn rudp_session_bounds_pending_reliable_packets_before_enqueue() -> Result<()> { let mut fragmented = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 103, - initial_sequence: 1, + initial_sequence: Some(1), max_pending_reliable_packets: Some(3), ..CultNetRudpSessionOptions::default() }); - fragmented.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 103, - sequence: 50, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + fragmented.assume_connected(0); let error = fragmented .send_many( "schema", @@ -995,50 +894,17 @@ fn rudp_session_suppresses_duplicates_and_delivers_reliable_ordered_payloads_in_ -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 123, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 123, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); - sender.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 123, - sequence: 90, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - receiver.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 123, - sequence: 91, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let options = CultNetRudpSendOptions { reliable: true, @@ -1076,50 +942,17 @@ fn rudp_session_suppresses_duplicates_and_delivers_reliable_ordered_payloads_in_ fn rudp_session_skips_control_packets_while_ordering_schema_payloads() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 124, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 124, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); - sender.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 124, - sequence: 90, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - receiver.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 124, - sequence: 91, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let options = CultNetRudpSendOptions { reliable: true, @@ -1158,50 +991,17 @@ fn rudp_session_skips_control_packets_while_ordering_schema_payloads() -> Result fn rudp_session_fragments_and_reassembles_reliable_ordered_payloads() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 456, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 456, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); - sender.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 456, - sequence: 90, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - receiver.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id: 456, - sequence: 91, - ack: 0, - ack_mask: 0, - channel_id: "control".to_string(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let packets = sender.send_many( "schema", @@ -1259,7 +1059,7 @@ fn rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames() mode: CultNetRudpSocketMode::Server, remote_addr: None, connection_id, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1276,7 +1076,7 @@ fn rudp_socket_transport_handshakes_and_carries_reliable_ordered_schema_frames() mode: CultNetRudpSocketMode::Client, remote_addr: Some(server_addr), connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1353,7 +1153,7 @@ fn rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames() -> mode: CultNetRudpSocketMode::Server, remote_addr: None, connection_id, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1370,7 +1170,7 @@ fn rudp_socket_transport_carries_fragmented_reliable_ordered_schema_frames() -> mode: CultNetRudpSocketMode::Client, remote_addr: Some(server_addr), connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1398,36 +1198,19 @@ fn rudp_session_advances_large_fragment_sets_through_a_bounded_reliable_window() let connection_id = 0x1020_3042; let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 25, max_pending_reliable_packets: None, }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 25, max_pending_reliable_packets: None, }); - for session in [&mut sender, &mut receiver] { - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id, - sequence: 0, - ack: 0, - ack_mask: 0, - channel_id: "control".into(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; - } + let connect = sender.create_connect(0, Vec::new())?; + let accept = receiver.accept_connect(&connect, 0, Vec::new())?; + sender.receive(&accept, 0)?; let fragment_count = CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + 17; let payload = (0..fragment_count * 8) @@ -1485,7 +1268,7 @@ fn rudp_socket_flush_waits_for_large_fragment_delivery() -> Result<()> { mode: CultNetRudpSocketMode::Server, remote_addr: None, connection_id, - initial_sequence: 100, + initial_sequence: Some(100), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -1502,7 +1285,7 @@ fn rudp_socket_flush_waits_for_large_fragment_delivery() -> Result<()> { mode: CultNetRudpSocketMode::Client, remote_addr: Some(server_addr), connection_id, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 25, transport_id: None, max_payload_bytes: None, @@ -2609,12 +2392,12 @@ fn schema_discovery_round_trips_over_legacy_gamecult_contract_when_inline_schema fn rudp_sequenced_channel_tracking_is_bounded() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 401, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 401, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -2649,18 +2432,30 @@ fn rudp_sequenced_channel_tracking_is_bounded() -> Result<()> { fn rudp_ordered_channel_tracking_is_bounded() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 402, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 402, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; let accept = receiver.accept_connect(&connect, 1, Vec::new())?; sender.receive(&accept, 2)?; + // Channels are tracked only while they hold a frame, so the bound is on + // channels held behind one missing sequence. + let gap = sender.send( + "gap", + b"lost".to_vec(), + CultNetRudpSendOptions { + reliable: true, + ordered: false, + ..CultNetRudpSendOptions::default() + }, + )?; + sender.receive(&receiver.create_ack_for(gap.sequence), 3)?; let mut error = None; for channel in 0..256u32 { let packet = sender.send( @@ -2673,7 +2468,8 @@ fn rudp_ordered_channel_tracking_is_bounded() -> Result<()> { }, )?; match receiver.receive(&packet, 3) { - Ok(_) => { + Ok(result) => { + assert!(result.delivered.is_empty(), "the gap holds every frame"); sender.receive(&receiver.create_ack_for(packet.sequence), 3)?; } Err(refused) => { @@ -2694,12 +2490,12 @@ fn rudp_ordered_channel_tracking_is_bounded() -> Result<()> { fn rudp_ordered_hold_buffer_is_bounded() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 403, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 403, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -2737,16 +2533,196 @@ fn rudp_ordered_hold_buffer_is_bounded() -> Result<()> { Ok(()) } +fn rudp_ordered_pair() -> Result<(CultNetRudpSession, CultNetRudpSession)> { + let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 410, + initial_sequence: Some(1), + ..CultNetRudpSessionOptions::default() + }); + let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 410, + initial_sequence: Some(500), + ..CultNetRudpSessionOptions::default() + }); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + client.receive(&accept, 0)?; + Ok((client, server)) +} + +fn rudp_send_options(ordered: bool) -> CultNetRudpSendOptions { + CultNetRudpSendOptions { + reliable: true, + ordered, + ..CultNetRudpSendOptions::default() + } +} + +fn rudp_delivered_names(delivered: &[cultnet_rs::CultNetRudpDeliveredFrame]) -> Vec { + delivered + .iter() + .map(|frame| String::from_utf8(frame.payload.clone()).unwrap()) + .collect() +} + +#[test] +fn rudp_ordered_frame_waits_for_a_gap_filled_by_another_channel() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let o1 = sender.send("schema", b"o1".to_vec(), rudp_send_options(true))?; + let u = sender.send("media", b"u".to_vec(), rudp_send_options(false))?; + let o2 = sender.send("schema", b"o2".to_vec(), rudp_send_options(true))?; + let o3 = sender.send("schema", b"o3".to_vec(), rudp_send_options(true))?; + let o4 = sender.send("schema", b"o4".to_vec(), rudp_send_options(true))?; + + assert_eq!(rudp_delivered_names(&receiver.receive(&o1, 1)?.delivered), ["o1"]); + // `u` is lost, so nothing behind it may be delivered. + assert!(receiver.receive(&o2, 2)?.delivered.is_empty()); + assert!(receiver.receive(&o3, 3)?.delivered.is_empty()); + assert!(receiver.receive(&o4, 4)?.delivered.is_empty()); + // The other channel's packet fills the gap and releases every ordered + // frame behind it in the same call. + assert_eq!( + rudp_delivered_names(&receiver.receive(&u, 5)?.delivered), + ["u", "o2", "o3", "o4"] + ); + Ok(()) +} + +#[test] +fn rudp_two_ordered_channels_release_each_other_in_sequence_order() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let a1 = sender.send("schema", b"A1".to_vec(), rudp_send_options(true))?; + let b1 = sender.send("other", b"B1".to_vec(), rudp_send_options(true))?; + let a2 = sender.send("schema", b"A2".to_vec(), rudp_send_options(true))?; + let a3 = sender.send("schema", b"A3".to_vec(), rudp_send_options(true))?; + + let mut delivered = rudp_delivered_names(&receiver.receive(&a1, 1)?.delivered); + delivered.extend(rudp_delivered_names(&receiver.receive(&a2, 2)?.delivered)); + delivered.extend(rudp_delivered_names(&receiver.receive(&a3, 3)?.delivered)); + assert_eq!(delivered, ["A1"], "B1 is missing, so A2 and A3 wait"); + assert_eq!( + rudp_delivered_names(&receiver.receive(&b1, 4)?.delivered), + ["B1", "A2", "A3"] + ); + Ok(()) +} + +#[test] +fn rudp_channel_first_used_after_other_traffic_loses_nothing() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let c1 = sender.send("late", b"C1".to_vec(), rudp_send_options(true))?; + let c2 = sender.send("late", b"C2".to_vec(), rudp_send_options(true))?; + let x = sender.send("schema", b"X".to_vec(), rudp_send_options(true))?; + + // X, then C2, arrive while C1 is still in flight. + let mut delivered = rudp_delivered_names(&receiver.receive(&x, 1)?.delivered); + delivered.extend(rudp_delivered_names(&receiver.receive(&c2, 2)?.delivered)); + assert!(delivered.is_empty(), "C1 is missing, so nothing is delivered"); + assert_eq!( + rudp_delivered_names(&receiver.receive(&c1, 3)?.delivered), + ["C1", "C2", "X"] + ); + Ok(()) +} + +#[test] +fn rudp_accept_seeds_the_watermark_of_the_connecting_side() -> Result<()> { + let (mut client, mut server) = rudp_ordered_pair()?; + let s1 = server.send("schema", b"s1".to_vec(), rudp_send_options(true))?; + let s2 = server.send("schema", b"s2".to_vec(), rudp_send_options(true))?; + assert!(client.receive(&s2, 1)?.delivered.is_empty()); + assert_eq!( + rudp_delivered_names(&client.receive(&s1, 2)?.delivered), + ["s1", "s2"] + ); + Ok(()) +} + +#[test] +fn rudp_reset_forgets_held_frames_and_reseeds_the_watermark_from_the_next_handshake() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + sender.send("schema", b"s1".to_vec(), rudp_send_options(true))?; + let s2 = sender.send("schema", b"s2".to_vec(), rudp_send_options(true))?; + assert!(receiver.receive(&s2, 1)?.delivered.is_empty()); + + receiver.reset_peer_state(); + let mut next = CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: 410, + initial_sequence: Some(100), + ..CultNetRudpSessionOptions::default() + }); + let accept = receiver.accept_connect(&next.create_connect(0, Vec::new())?, 2, Vec::new())?; + next.receive(&accept, 2)?; + let d1 = next.send("schema", b"d1".to_vec(), rudp_send_options(true))?; + let d2 = next.send("schema", b"d2".to_vec(), rudp_send_options(true))?; + + // The held s2 died with the reset, and the watermark follows the new peer. + assert!(receiver.receive(&d2, 3)?.delivered.is_empty()); + assert_eq!(rudp_delivered_names(&receiver.receive(&d1, 4)?.delivered), ["d1", "d2"]); + Ok(()) +} + +#[test] +fn rudp_large_ordered_frame_without_a_gap_is_never_held_or_refused() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let packets = sender.send_many( + "schema", + vec![7u8; 5 * 1024 * 1024], + rudp_send_options(true), + Some(60_000), + )?; + let mut queue = packets; + let mut delivered = Vec::new(); + while !queue.is_empty() { + let mut next = Vec::new(); + for packet in &queue { + delivered.extend(receiver.receive(packet, 1)?.delivered); + let ack = receiver.create_ack_for_received(packet.sequence); + next.extend(sender.receive(&ack, 1)?.ready_to_send); + } + queue = next; + } + assert_eq!(delivered.len(), 1); + assert_eq!(delivered[0].payload.len(), 5 * 1024 * 1024); + Ok(()) +} + +#[test] +fn rudp_ordered_hold_buffer_is_bounded_in_bytes() -> Result<()> { + let (mut sender, mut receiver) = rudp_ordered_pair()?; + let withheld = sender.send("schema", b"first".to_vec(), rudp_send_options(true))?; + sender.receive(&receiver.create_ack_for(withheld.sequence), 1)?; + + // Four 1 MiB frames fit the 4 MiB hold; the fifth does not. + let mut refused_at = None; + for step in 0..8u32 { + let packet = sender.send("schema", vec![7u8; 1024 * 1024], rudp_send_options(true))?; + match receiver.receive(&packet, 2) { + Ok(result) => { + assert!(result.delivered.is_empty(), "the gap holds every frame"); + sender.receive(&receiver.create_ack_for(packet.sequence), 2)?; + } + Err(refusal) => { + assert!(refusal.to_string().contains("ordered hold buffer is full"), "{refusal}"); + refused_at = Some(step); + break; + } + } + } + assert_eq!(refused_at, Some(4)); + Ok(()) +} + #[test] fn rudp_far_ahead_sequence_cannot_starve_the_session() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 404, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 404, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -2795,12 +2771,12 @@ fn rudp_reliable_sequence_exhaustion_is_reported_not_fatal() -> Result<()> { connection_id: 405, // One sequence left: the handshake takes it, so the next send is the // first allocation with nowhere to go. - initial_sequence: u32::MAX - 1, + initial_sequence: Some(u32::MAX - 1), ..CultNetRudpSessionOptions::default() }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 405, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = sender.create_connect(0, Vec::new())?; @@ -2864,7 +2840,7 @@ fn tcp_framed_receive_refuses_a_length_beyond_the_advertised_maximum() -> Result fn rudp_reliable_expiry_drops_late_packets_and_leaves_durable_sends_alone() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 7, - initial_sequence: 10, + initial_sequence: Some(10), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); @@ -2928,7 +2904,7 @@ fn rudp_reliable_expiry_drops_late_packets_and_leaves_durable_sends_alone() -> R fn rudp_reliable_expiry_also_reclaims_the_send_backlog() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 9, - initial_sequence: 10, + initial_sequence: Some(10), resend_delay_ms: 100, ..CultNetRudpSessionOptions::default() }); @@ -2977,25 +2953,8 @@ fn rudp_reliable_expiry_also_reclaims_the_send_backlog() -> Result<()> { Ok(()) } -fn accept_rudp_session(session: &mut CultNetRudpSession, connection_id: u32) -> Result<()> { - session.receive( - &CultNetRudpPacket { - packet_type: CultNetRudpPacketType::Accept, - connection_id, - sequence: 0, - ack: 0, - ack_mask: 0, - channel_id: "control".into(), - reliable: false, - ordered: false, - sequenced: false, - fragment_id: 0, - fragment_index: 0, - fragment_count: 0, - payload: Vec::new(), - }, - 0, - )?; +fn accept_rudp_session(session: &mut CultNetRudpSession, _connection_id: u32) -> Result<()> { + session.assume_connected(0); Ok(()) } @@ -3032,12 +2991,12 @@ fn rudp_profile_expires_only_the_media_channel() -> Result<()> { fn connected_rudp_pair() -> Result<(CultNetRudpSession, CultNetRudpSession)> { let mut client = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 311, - initial_sequence: 1, + initial_sequence: Some(1), ..CultNetRudpSessionOptions::default() }); let mut server = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: 311, - initial_sequence: 100, + initial_sequence: Some(100), ..CultNetRudpSessionOptions::default() }); let connect = client.create_connect(0, Vec::new())?; diff --git a/packages/cultnet-rs/tests/rudp_connect_generation.rs b/packages/cultnet-rs/tests/rudp_connect_generation.rs new file mode 100644 index 00000000..b456162c --- /dev/null +++ b/packages/cultnet-rs/tests/rudp_connect_generation.rs @@ -0,0 +1,1001 @@ +//! A Connect's sequence says whether it repeats the one a session accepted, and +//! the handshake alone seeds the watermark that orders delivery. Each test puts +//! one rule in front of real sessions or real transports. + +use std::collections::BTreeSet; +use std::net::{SocketAddr, UdpSocket}; +use std::time::Duration; + +use anyhow::{Result, anyhow}; +use cultnet_rs::*; + +const CONNECTION_ID: u32 = 0x4355_4c54; + +fn socket() -> Result { + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.set_read_timeout(Some(Duration::from_millis(20)))?; + Ok(socket) +} + +fn session(initial_sequence: u32) -> CultNetRudpSession { + CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + initial_sequence: Some(initial_sequence), + resend_delay_ms: 250, + max_pending_reliable_packets: None, + }) +} + +fn send(from: &mut CultNetRudpSession, text: &str) -> Result { + from.send("schema", text.as_bytes().to_vec(), ordered()) +} + +fn ordered() -> CultNetRudpSendOptions { + CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..Default::default() + } +} + +fn names(frames: &[CultNetRudpDeliveredFrame]) -> Vec { + frames + .iter() + .map(|frame| String::from_utf8(frame.payload.clone()).unwrap()) + .collect() +} + +/// A completed handshake between two sessions; returns the Connect and Accept. +fn handshake( + client: &mut CultNetRudpSession, + server: &mut CultNetRudpSession, +) -> Result<(CultNetRudpPacket, CultNetRudpPacket)> { + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + client.receive(&accept, 0)?; + assert!(client.connected() && server.connected()); + Ok((connect, accept)) +} + +/// What the server owed the client when the session ended is gone. A client +/// that reconnects on the same session must still receive what the new server +/// session sends: the old session's lost write is not a hole in the new one. +#[test] +fn a_reconnect_on_the_same_session_is_not_stranded_by_a_write_the_ended_session_owed() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let _lost = server.send("schema", b"lost".to_vec(), ordered())?; + server.create_disconnect(b"gone".to_vec()); + + let connect = client.create_connect(1, Vec::new())?; + let accept = server.accept_connect(&connect, 1, Vec::new())?; + client.receive(&accept, 1)?; + let after = server.send("schema", b"after".to_vec(), ordered())?; + assert_eq!(names(&client.receive(&after, 2)?.delivered), ["after"]); + Ok(()) +} + +/// A server session that ended and then accepts a Connect from a new client +/// forgets what the old client sent: the new client's sequences are its own. +#[test] +fn a_server_that_accepts_a_new_client_forgets_the_old_clients_sequences() -> Result<()> { + let mut old_client = session(50); + let mut server = session(500); + handshake(&mut old_client, &mut server)?; + for name in ["c1", "c2", "c3"] { + let packet = old_client.send("schema", name.as_bytes().to_vec(), ordered())?; + assert_eq!(names(&server.receive(&packet, 1)?.delivered), [name]); + } + server.create_disconnect(b"restart".to_vec()); + + let mut new_client = session(2); + let connect = new_client.create_connect(2, Vec::new())?; + let accept = server.accept_connect(&connect, 2, Vec::new())?; + new_client.receive(&accept, 2)?; + let frame = new_client.send("schema", b"fresh".to_vec(), ordered())?; + assert_eq!(names(&server.receive(&frame, 3)?.delivered), ["fresh"]); + Ok(()) +} + +/// The same Connect again is a retransmit: it does not reset the session, so +/// what the session already delivered stays delivered once. +#[test] +fn a_retransmitted_connect_after_data_has_flowed_does_not_reset_the_session() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let (connect, _) = handshake(&mut client, &mut server)?; + let frame = client.send("schema", b"once".to_vec(), ordered())?; + assert_eq!(names(&server.receive(&frame, 1)?.delivered), ["once"]); + + assert!(server.connect_repeats(&connect)); + let reply = server.accept_connect(&connect, 2, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack, "the Accept was already acknowledged"); + assert_eq!(server.outstanding_reliable_packet_count(), 0, "a repeat queued something"); + assert!( + server.receive(&frame, 3)?.delivered.is_empty(), + "a retransmitted Connect made the server forget what it had delivered" + ); + Ok(()) +} + +/// A Connect that carries another sequence is not a repeat, even from a peer +/// that sent the same bytes. +#[test] +fn a_connect_with_another_sequence_is_not_a_repeat_whatever_it_carries() -> Result<()> { + let mut server = session(500); + let mut first = session(1); + let connect = first.create_connect(0, b"same payload".to_vec())?; + server.accept_connect(&connect, 0, Vec::new())?; + assert!(server.connect_repeats(&connect)); + + let mut restarted = session(9); + let again = restarted.create_connect(0, b"same payload".to_vec())?; + assert!(!server.connect_repeats(&again)); + let mut not_a_connect = connect.clone(); + not_a_connect.packet_type = CultNetRudpPacketType::Ping; + assert!(!server.connect_repeats(¬_a_connect)); + server.create_disconnect(Vec::new()); + assert!(!server.connect_repeats(&connect), "an ended session repeats nothing"); + Ok(()) +} + +/// An Accept counts only when its ack field names the pending Connect. +#[test] +fn an_accept_that_does_not_name_the_pending_connect_is_ignored() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + + let mut stale = accept.clone(); + stale.ack = connect.sequence + 50; + stale.ack_mask = 0; + client.receive(&stale, 1)?; + assert!(!client.connected(), "a stale Accept connected the session"); + assert_eq!(client.pending_reliable_sequences(), vec![connect.sequence]); + + let mut named_by_mask = accept.clone(); + named_by_mask.ack = connect.sequence + 3; + named_by_mask.ack_mask = 1 << 2; + client.receive(&named_by_mask, 2)?; + assert!(client.connected(), "the mask names the Connect"); + Ok(()) +} + +/// A late or duplicate Accept after a local disconnect must not revive the +/// ended session. +#[test] +fn an_accept_after_a_local_disconnect_does_not_reconnect_the_session() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let (_, accept) = handshake(&mut client, &mut server)?; + client.create_disconnect(b"bye".to_vec()); + assert!(!client.connected()); + + client.receive(&accept, 5)?; + assert!(!client.connected(), "a late Accept revived an ended session"); + Ok(()) +} + +/// A client that reconnects to a restarted server is not silenced by what it +/// received from the old one: the new server's sequences may overlap the old +/// ones, and none of them is a duplicate. +#[test] +fn a_reconnect_forgets_what_the_old_server_sent() -> Result<()> { + let mut client = session(1); + let mut old_server = session(500); + handshake(&mut client, &mut old_server)?; + for name in ["d1", "d2", "d3"] { + let packet = send(&mut old_server, name)?; + assert_eq!(names(&client.receive(&packet, 1)?.delivered), [name]); + } + + let mut new_server = session(501); + let connect = client.create_connect(2, Vec::new())?; + let accept = new_server.accept_connect(&connect, 2, Vec::new())?; + client.receive(&accept, 2)?; + let fresh = send(&mut new_server, "fresh")?; + assert_eq!(names(&client.receive(&fresh, 3)?.delivered), ["fresh"]); + Ok(()) +} + +/// An Accept for a Connect that was abandoned before it was answered does not +/// connect the session. +#[test] +fn an_accept_for_an_abandoned_connect_does_not_connect_the_session() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + client.create_disconnect(b"never mind".to_vec()); + + client.receive(&accept, 1)?; + assert!(!client.connected(), "an Accept revived an abandoned Connect"); + Ok(()) +} + +/// What a session still owed a vanished peer does not keep the next peer out: +/// a new generation drops it before the queue is asked for room. +#[test] +fn a_full_queue_owed_to_a_vanished_peer_does_not_refuse_a_new_generation() -> Result<()> { + let bounded = |initial_sequence: u32| { + CultNetRudpSession::new(CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + initial_sequence: Some(initial_sequence), + resend_delay_ms: 250, + max_pending_reliable_packets: Some(1), + }) + }; + let mut server = bounded(500); + server.accept_connect(&session(1).create_connect(0, Vec::new())?, 0, Vec::new())?; + assert_eq!(server.outstanding_reliable_packet_count(), 1, "the Accept is owed and never acknowledged"); + let next = session(9).create_connect(1, Vec::new())?; + server.accept_connect(&next, 1, Vec::new())?; + assert!(server.connect_repeats(&next)); + + let mut client = bounded(1); + client.create_connect(0, Vec::new())?; + client.create_connect(1, Vec::new())?; + assert_eq!(client.outstanding_reliable_packet_count(), 1, "only the new Connect is owed"); + Ok(()) +} + +/// An Accept counts once: a duplicate of the one honoured does not seed the +/// watermark again. +#[test] +fn a_duplicate_accept_does_not_reseed_the_watermark() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let (_, accept) = handshake(&mut client, &mut server)?; + let s1 = server.send("schema", b"s1".to_vec(), ordered())?; + let s2 = server.send("schema", b"s2".to_vec(), ordered())?; + let mut forged = accept.clone(); + forged.sequence = s2.sequence; + client.receive(&forged, 1)?; + assert!(client.receive(&s2, 2)?.delivered.is_empty(), "s1 is still missing"); + assert_eq!(names(&client.receive(&s1, 3)?.delivered), ["s1", "s2"]); + Ok(()) +} + +/// Reliable data that arrives before the connecting side has the Accept has no +/// place in the order: it is not delivered and not acknowledged, so the sender +/// retransmits it after the handshake. +#[test] +fn reliable_data_before_the_accept_is_neither_delivered_nor_acknowledged() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + let connect = client.create_connect(0, Vec::new())?; + let accept = server.accept_connect(&connect, 0, Vec::new())?; + let a = server.send("schema", b"A".to_vec(), ordered())?; + let b = server.send("schema", b"B".to_vec(), ordered())?; + + for early in [&b, &a] { + assert!(client.receive(early, 1)?.delivered.is_empty()); + let ack = client.create_ack_for_received(early.sequence); + assert_eq!((ack.ack, ack.ack_mask), (0, 0), "an unhandled packet was acknowledged"); + } + client.receive(&accept, 2)?; + assert!(client.connected()); + assert_eq!(server.outstanding_reliable_packet_count(), 3, "nothing was acknowledged"); + + assert_eq!(names(&client.receive(&a, 3)?.delivered), ["A"]); + assert_eq!(names(&client.receive(&b, 3)?.delivered), ["B"]); + Ok(()) +} + +/// The other order: with the Accept in hand, frames that arrive out of order +/// are delivered in order. +#[test] +fn ordered_frames_after_the_accept_are_delivered_in_order_whatever_arrives_first() -> Result<()> { + let mut client = session(1); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let a = server.send("schema", b"A".to_vec(), ordered())?; + let b = server.send("schema", b"B".to_vec(), ordered())?; + assert!(client.receive(&b, 1)?.delivered.is_empty()); + assert_eq!(names(&client.receive(&a, 2)?.delivered), ["A", "B"]); + Ok(()) +} + +/// The options say nothing about the initial sequence; each session draws its +/// own, so one options object serves many sessions without giving them one +/// sequence. +#[test] +fn sessions_from_one_options_object_each_draw_their_own_initial_sequence() -> Result<()> { + fn assert_random(label: &str, draws: &BTreeSet) { + assert!(draws.len() > 32, "{label} does not draw at random: {draws:?}"); + assert!( + draws.iter().all(|value| (1..(1_u32 << 31)).contains(value)), + "{label} left [1, 2^31): {draws:?}" + ); + } + let address: SocketAddr = "127.0.0.1:9".parse()?; + assert_eq!(CultNetRudpSessionOptions::default().initial_sequence, None); + assert_eq!( + CultNetRudpSocketTransportOptions::client("c", socket()?, address, 1).initial_sequence, + None + ); + assert_eq!( + CultNetRudpSocketTransportOptions::server("s", socket()?, 1).initial_sequence, + None + ); + assert_eq!( + CultNetRudpServerHubOptions::new("h", socket()?, 1).initial_sequence, + None + ); + assert_eq!(CultMeshRudpSocketOptions::default().initial_sequence, None); + + let options = CultNetRudpSessionOptions { + connection_id: CONNECTION_ID, + ..Default::default() + }; + let connects: BTreeSet = (0..64) + .map(|_| Ok(CultNetRudpSession::new(options.clone()).create_connect(0, Vec::new())?.sequence)) + .collect::>()?; + assert_random("sessions from one options object", &connects); + Ok(()) +} + +/// A transport built from the same options each time (a reconnect loop's +/// factory) opens each connection with its own Connect sequence. +#[test] +fn a_reused_options_object_yields_a_different_connect_sequence_for_each_transport() -> Result<()> { + let listener = socket()?; + let endpoint = CultNetRudpEndpoint { host: "127.0.0.1".into(), port: listener.local_addr()?.port() }; + let options = CultMeshRudpSocketOptions::default(); + let mut sequences = BTreeSet::new(); + for _ in 0..8 { + let mut client = + CultMesh::create_rudp_client("client", CONNECTION_ID, &endpoint, options.clone())?; + client.connect(Vec::new())?; + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = listener.recv_from(&mut buffer)?; + sequences.insert(decode_rudp_packet(&buffer[..received])?.sequence); + } + assert_eq!(sequences.len(), 8, "two connections opened with one Connect sequence: {sequences:?}"); + Ok(()) +} + +/// A hub builds a session for every client it admits; each draws its own +/// sequence, so no two clients are accepted with the same one. +#[test] +fn a_hub_accepts_each_client_with_its_own_sequence() -> Result<()> { + let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID))?; + let hub_addr = hub.local_addr()?; + let mut accepts = BTreeSet::new(); + for _ in 0..8 { + let peer = socket()?; + let connect = session(1).create_connect(0, Vec::new())?; + peer.send_to(&encode_rudp_packet(&connect)?, hub_addr)?; + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))?; + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = peer.recv_from(&mut buffer)?; + accepts.insert(decode_rudp_packet(&buffer[..received])?.sequence); + } + assert_eq!(accepts.len(), 8, "two clients were accepted with one sequence: {accepts:?}"); + Ok(()) +} + +// Transports. + +fn hub_event_matching( + hub: &mut CultNetRudpServerHub, + matches: impl Fn(&CultNetRudpServerEvent) -> bool, +) -> Result { + for _ in 0..40 { + if let Some(event) = hub.receive_event_once()? + && matches(&event) + { + return Ok(event); + } + } + Err(anyhow!("the hub never produced the event")) +} + +fn client_on( + socket: UdpSocket, + remote: SocketAddr, + initial_sequence: u32, +) -> Result { + let mut options = CultNetRudpSocketTransportOptions::client("client", socket, remote, CONNECTION_ID); + options.initial_sequence = Some(initial_sequence); + CultNetRudpSocketTransportConnection::new(options) +} + +/// A client that restarts on the same address, connection id and Connect +/// payload is a new session, not a retransmit: the hub reports the old one +/// ended and delivers the new one's frames. +#[test] +fn the_hub_admits_a_restarted_client_with_the_same_address_id_and_payload() -> Result<()> { + let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID))?; + let hub_addr = hub.local_addr()?; + let shared = socket()?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, hub_addr, 50_000)?; + first.connect(b"same".to_vec())?; + let CultNetRudpServerEvent::Connected { session: original } = + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))? + else { + unreachable!() + }; + first.receive_once()?; + for index in 0..3_u8 { + first.send("schema", vec![index])?; + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Frame { .. }))?; + } + + let mut second = client_on(twin, hub_addr, 7)?; + second.connect(b"same".to_vec())?; + let CultNetRudpServerEvent::Disconnected { session: ended, .. } = + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Disconnected { .. }))? + else { + unreachable!() + }; + assert_eq!(ended, original); + let CultNetRudpServerEvent::Connected { session: replacement } = + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))? + else { + unreachable!() + }; + assert_ne!(replacement.session_generation, original.session_generation); + second.receive_once()?; + second.send("schema", b"after restart".to_vec())?; + let CultNetRudpServerEvent::Frame { session, frame } = + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Frame { .. }))? + else { + unreachable!() + }; + assert_eq!(session, replacement); + assert_eq!(frame.payload, b"after restart"); + Ok(()) +} + +/// The same for a server-mode transport, where the restarted client's sequences +/// are all below what the old client used. +#[test] +fn server_mode_admits_a_restarted_client_and_delivers_its_frames() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions::server( + "server", + server_socket, + CONNECTION_ID, + ))?; + let shared = socket()?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, server_addr, 50_000)?; + first.connect(b"same".to_vec())?; + assert!(server.receive_once()?.is_none()); + first.receive_once()?; + for index in 0..3_u8 { + first.send("schema", vec![index])?; + assert_eq!(server.receive_once()?.expect("first client frame").payload, vec![index]); + } + + let mut second = client_on(twin, server_addr, 7)?; + second.connect(b"same".to_vec())?; + assert!(server.receive_once()?.is_none()); + second.receive_once()?; + assert!(second.connected()); + second.send("schema", b"after restart".to_vec())?; + assert_eq!(server.receive_once()?.expect("restarted client frame").payload, b"after restart"); + Ok(()) +} + +/// A retransmitted Connect in server mode, after data has flowed, keeps the +/// session: the data it delivered is not delivered again. +#[test] +fn server_mode_keeps_its_session_when_a_connect_is_retransmitted_after_data_flowed() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions::server( + "server", + server_socket, + CONNECTION_ID, + ))?; + let peer = socket()?; + let mut client = session(1); + let connect = client.create_connect(0, b"peer".to_vec())?; + peer.send_to(&encode_rudp_packet(&connect)?, server_addr)?; + server.receive_once()?; + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = peer.recv_from(&mut buffer)?; + client.receive(&decode_rudp_packet(&buffer[..received])?, 0)?; + let frame = client.send("schema", b"once".to_vec(), ordered())?; + peer.send_to(&encode_rudp_packet(&frame)?, server_addr)?; + assert_eq!(server.receive_once()?.expect("first delivery").payload, b"once"); + + peer.send_to(&encode_rudp_packet(&connect)?, server_addr)?; + server.receive_once()?; + peer.send_to(&encode_rudp_packet(&frame)?, server_addr)?; + assert!(server.receive_once()?.is_none(), "the retransmitted Connect reset the session"); + Ok(()) +} + +fn connected_client( + server_socket: &UdpSocket, + server_addr: SocketAddr, +) -> Result { + let mut client = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions::client( + "client", + socket()?, + server_addr, + CONNECTION_ID, + ))?; + client.connect(b"hello".to_vec())?; + let mut buffer = vec![0_u8; 65_535]; + // Another client's acknowledgements may be queued ahead of this Connect. + let (connect, client_addr) = loop { + let (received, from) = server_socket.recv_from(&mut buffer)?; + let packet = decode_rudp_packet(&buffer[..received])?; + if packet.packet_type == CultNetRudpPacketType::Connect { + break (packet, from); + } + }; + let accept = session(500).accept_connect(&connect, 0, Vec::new())?; + server_socket.send_to(&encode_rudp_packet(&accept)?, client_addr)?; + client.receive_once()?; + assert!(client.connected()); + Ok(client) +} + +/// A receipt belongs to the transport that issued it: asked of another +/// transport, it is not that transport's write, however alike their numbers. +#[test] +fn a_receipt_asked_of_another_transport_reads_invalidated() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let mut a = connected_client(&server, server_addr)?; + let mut b = connected_client(&server, server_addr)?; + let receipt_a = a.send_reliable("schema", b"a".to_vec())?; + let receipt_b = b.send_reliable("schema", b"b".to_vec())?; + assert_eq!(a.reliable_send_status(&receipt_a), CultNetRudpReliableSendStatus::Pending); + assert_eq!(b.reliable_send_status(&receipt_b), CultNetRudpReliableSendStatus::Pending); + assert_eq!(b.reliable_send_status(&receipt_a), CultNetRudpReliableSendStatus::Invalidated); + assert_eq!(a.reliable_send_status(&receipt_b), CultNetRudpReliableSendStatus::Invalidated); + assert_ne!(receipt_a, receipt_b, "receipts from two transports compare equal"); + Ok(()) +} + +/// A flush cut short by a timeout says so. +#[test] +fn a_flush_that_a_timeout_ended_names_the_timeout() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let mut client = connected_client(&server, server_addr)?; + client.send_reliable("schema", b"unanswered".to_vec())?; + std::thread::sleep(Duration::from_millis(5)); + assert!(client.check_timeout(0)); + let error = client.flush_reliable(Duration::from_millis(50)).expect_err("the session timed out"); + assert!(error.to_string().contains("session timed out"), "{error}"); + Ok(()) +} + +// Hold-buffer boundaries: 1,024 frames and 64 channels are held; one more is +// refused. + +/// A connected pair with one sequence the receiver will never see, so every +/// later ordered frame waits behind it. The sender's window is kept open by +/// acknowledging what the receiver has. +fn held_behind_a_gap( + frames: usize, + channel_of: impl Fn(usize) -> String, +) -> Result> { + let mut sender = session(1); + let mut receiver = session(500); + handshake(&mut sender, &mut receiver)?; + let _missing = sender.send("gap", b"lost".to_vec(), ordered())?; + let mut held = 0; + for index in 0..frames { + let packet = sender.send(&channel_of(index), vec![0], ordered())?; + match receiver.receive(&packet, 1) { + Ok(result) => { + assert!(result.delivered.is_empty()); + held += 1; + } + Err(error) => return Ok(Err(error)), + } + if index % 16 == 15 { + sender.receive(&receiver.create_ack(), 1)?; + } + } + Ok(Ok(held)) +} + +#[test] +fn the_hold_buffer_takes_exactly_1024_frames_and_refuses_the_next() -> Result<()> { + assert_eq!(held_behind_a_gap(1024, |_| "schema".into())??, 1024); + let error = held_behind_a_gap(1025, |_| "schema".into())?.expect_err("1025 held frames"); + assert!(error.to_string().contains("hold buffer is full"), "{error}"); + Ok(()) +} + +#[test] +fn the_hold_buffer_takes_exactly_64_channels_and_refuses_the_next() -> Result<()> { + assert_eq!(held_behind_a_gap(64, |index| format!("channel-{index}"))??, 64); + let error = held_behind_a_gap(65, |index| format!("channel-{index}"))?.expect_err("65 held channels"); + assert!(error.to_string().contains("too many ordered channels"), "{error}"); + Ok(()) +} + +// Ack Cut 1d: a frame from an earlier generation is a duplicate, a delayed +// Connect from an earlier attempt is stale, a Connect from a new endpoint is a +// new client. + +/// Whether an acknowledgement names `sequence` by its ack field or its mask. +fn names_sequence(ack: &CultNetRudpPacket, sequence: u32) -> bool { + ack.ack == sequence + || (0..32).any(|bit| { + ack.ack_mask & (1_u32 << bit) != 0 && ack.ack > bit && ack.ack - bit - 1 == sequence + }) +} + +/// A frame the client's old generation already delivered, retransmitted after +/// the client reconnected, is acknowledged and not delivered into the new +/// generation. Every sequence the old generation issued is below the new +/// Connect. +#[test] +fn a_frame_from_the_clients_earlier_generation_is_acknowledged_not_delivered_again() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let old = send(&mut client, "old")?; + assert_eq!(names(&server.receive(&old, 1)?.delivered), ["old"]); + + let connect = client.create_connect(2, Vec::new())?; + let accept = server.accept_connect(&connect, 2, Vec::new())?; + client.receive(&accept, 2)?; + assert!(old.sequence < connect.sequence); + let late = server.receive(&old, 3)?; + assert!(late.delivered.is_empty(), "an earlier generation's frame was delivered again"); + let ack = server.create_ack_for_received(old.sequence); + assert!(names_sequence(&ack, old.sequence), "the duplicate was not acknowledged: {ack:?}"); + + let fresh = send(&mut client, "fresh")?; + assert_eq!(names(&server.receive(&fresh, 4)?.delivered), ["fresh"]); + Ok(()) +} + +/// The other direction: the server's frame delivered before the client +/// reconnected is not delivered to the client's new generation. +#[test] +fn a_frame_from_the_servers_earlier_generation_is_acknowledged_not_delivered_again() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let old = send(&mut server, "old")?; + assert_eq!(names(&client.receive(&old, 1)?.delivered), ["old"]); + + let connect = client.create_connect(2, Vec::new())?; + let accept = server.accept_connect(&connect, 2, Vec::new())?; + client.receive(&accept, 2)?; + let late = client.receive(&old, 3)?; + assert!(late.delivered.is_empty(), "an earlier generation's frame was delivered again"); + assert!(names_sequence(&client.create_ack_for_received(old.sequence), old.sequence)); + + let fresh = send(&mut server, "fresh")?; + assert_eq!(names(&client.receive(&fresh, 4)?.delivered), ["fresh"]); + Ok(()) +} + +/// A frame far below the receiver's highest sequence is a duplicate that is +/// acknowledged by name; without the acknowledgement its sender retransmits it +/// for ever. +#[test] +fn a_duplicate_below_the_receive_window_is_acknowledged_by_name() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + handshake(&mut client, &mut server)?; + let first = send(&mut client, "first")?; + assert_eq!(names(&server.receive(&first, 1)?.delivered), ["first"]); + for _ in 0..4_200 { + let packet = send(&mut client, "x")?; + server.receive(&packet, 1)?; + client.receive(&server.create_ack(), 1)?; + } + let again = server.receive(&first, 2)?; + assert!(again.delivered.is_empty()); + let ack = server.create_ack_for_received(first.sequence); + assert_eq!(ack.ack, first.sequence, "a duplicate below the window was not acknowledged by name"); + Ok(()) +} + +/// A Connect delayed from an earlier attempt of the client that is now +/// connected does not restart the session: the server answers with an Ack and +/// the client's ordered frames keep flowing. +#[test] +fn a_delayed_connect_from_an_earlier_attempt_does_not_strand_the_client() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + let earlier = client.create_connect(0, Vec::new())?; + let retried = client.create_connect(300, Vec::new())?; + let accept = server.accept_connect(&retried, 301, Vec::new())?; + client.receive(&accept, 302)?; + server.receive(&client.create_ack_for_received(accept.sequence), 302)?; + assert!(client.connected()); + let a = send(&mut client, "a")?; + assert_eq!(names(&server.receive(&a, 303)?.delivered), ["a"]); + client.receive(&server.create_ack_for_received(a.sequence), 303)?; + + assert!(server.connect_repeats(&earlier), "the delayed Connect starts nothing"); + let reply = server.accept_connect(&earlier, 304, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + client.receive(&reply, 305)?; + assert!(client.connected()); + + let b = send(&mut client, "b")?; + assert_eq!(names(&server.receive(&b, 306)?.delivered), ["b"], "the delayed Connect reset the server"); + client.receive(&server.create_ack_for_received(b.sequence), 307)?; + assert!(!client.pending_reliable_sequences().contains(&b.sequence)); + Ok(()) +} + +/// A stale Connect is the peer's echo from the past, not evidence it is alive; +/// a retransmit of the accepted Connect is. +#[test] +fn a_repeated_connect_refreshes_liveness_and_a_stale_one_does_not() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + let earlier = client.create_connect(0, Vec::new())?; + let current = client.create_connect(1, Vec::new())?; + server.accept_connect(¤t, 0, Vec::new())?; + + server.accept_connect(¤t, 900, Vec::new())?; + assert!(!server.check_timeout(1_000, 500), "a repeated Connect did not refresh liveness"); + + server.accept_connect(&earlier, 1_400, Vec::new())?; + assert!(server.check_timeout(1_600, 500), "a stale Connect refreshed liveness"); + Ok(()) +} + +/// The window is serial: a sequence just before the current Connect's is stale +/// across the wrap of the 32-bit space, one just after is a new client. +#[test] +fn stale_connects_are_recognised_by_serial_arithmetic() -> Result<()> { + let mut server = session(500); + let current = session(3).create_connect(0, Vec::new())?; + server.accept_connect(¤t, 0, Vec::new())?; + let before_the_wrap = session(u32::MAX - 1).create_connect(0, Vec::new())?; + assert!(server.connect_repeats(&before_the_wrap)); + let after = session(4).create_connect(0, Vec::new())?; + assert!(!server.connect_repeats(&after)); + let beyond_the_window = session(3 + 4_096).create_connect(0, Vec::new())?; + assert!(!server.connect_repeats(&beyond_the_window)); + let just_inside = session(3_u32.wrapping_sub(4_095)).create_connect(0, Vec::new())?; + assert!(server.connect_repeats(&just_inside)); + let just_outside = session(3_u32.wrapping_sub(4_096)).create_connect(0, Vec::new())?; + assert!(!server.connect_repeats(&just_outside)); + Ok(()) +} + +/// A restarted client whose random first sequence lands just before the +/// server's current Connect is answered with an Ack, never an Accept. It does +/// not retransmit that Connect for ever: once the attempt times out it starts +/// a fresh one with a newly drawn sequence, and is admitted. +#[test] +fn a_restarted_client_whose_first_sequence_is_stale_connects_after_redrawing() -> Result<()> { + let mut server = session(500); + let mut old = session(1_000); + let connect = old.create_connect(0, Vec::new())?; + let old_accept = server.accept_connect(&connect, 0, Vec::new())?; + old.receive(&old_accept, 0)?; + + let mut restarted = session(900); + let first = restarted.create_connect(0, b"join".to_vec())?; + // The old client has not acknowledged its Accept yet, so that is the reply: + // it names the old Connect, not this one. + let reply = server.accept_connect(&first, 1, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Accept); + restarted.receive(&reply, 1)?; + assert!(!restarted.connected(), "an Accept for another Connect connected the client"); + server.receive(&old.create_ack_for_received(old_accept.sequence), 1)?; + let reply = server.accept_connect(&first, 2, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + restarted.receive(&reply, 2)?; + assert!(!restarted.connected()); + + let retransmitted = restarted.due_resends(1_000); + assert_eq!(retransmitted.len(), 1); + assert_eq!(retransmitted[0].sequence, first.sequence, "the attempt is still young"); + + let fresh = restarted.due_resends(3_500); + assert_eq!(fresh.len(), 1); + assert_eq!(fresh[0].packet_type, CultNetRudpPacketType::Connect); + assert_ne!(fresh[0].sequence, first.sequence, "the same Connect was retransmitted for ever"); + assert_eq!(fresh[0].payload, b"join", "the fresh attempt lost the Connect payload"); + + let accept = server.accept_connect(&fresh[0], 3_500, Vec::new())?; + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + restarted.receive(&accept, 3_500)?; + assert!(restarted.connected()); + let frame = send(&mut restarted, "hello")?; + assert_eq!(names(&server.receive(&frame, 3_501)?.delivered), ["hello"]); + Ok(()) +} + +/// A client whose Connect was answered is not given a new Connect by the +/// clock: the attempt timeout belongs to an unanswered Connect. +#[test] +fn an_answered_connect_is_never_replaced_by_the_attempt_timeout() -> Result<()> { + let mut client = session(10); + let mut server = session(500); + let (connect, _) = handshake(&mut client, &mut server)?; + let resent = client.due_resends(60_000); + assert!( + resent.iter().all(|packet| packet.packet_type != CultNetRudpPacketType::Connect + || packet.sequence == connect.sequence), + "a connected client started a new attempt" + ); + assert!(client.connected()); + Ok(()) +} + +/// A pinned client that restarts on a new port against a server-mode transport +/// is a new client: the Connect starts a new generation and the client is +/// admitted, though its sequence is the one the old client's Connect carried. +#[test] +fn server_mode_admits_a_pinned_client_that_restarts_on_a_new_port() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new(CultNetRudpSocketTransportOptions::server( + "server", + server_socket, + CONNECTION_ID, + ))?; + let mut first = client_on(socket()?, server_addr, 1)?; + first.connect(b"join".to_vec())?; + assert!(server.receive_once()?.is_none()); + first.receive_once()?; + first.send("schema", b"hello".to_vec())?; + assert_eq!(server.receive_once()?.expect("first client frame").payload, b"hello"); + drop(first); + + let mut second = client_on(socket()?, server_addr, 1)?; + second.connect(b"join".to_vec())?; + assert!(server.receive_once()?.is_none()); + second.receive_once()?; + assert!(second.connected(), "the restarted client on a new port was not admitted"); + second.send("schema", b"after".to_vec())?; + assert_eq!(server.receive_once()?.expect("restarted client frame").payload, b"after"); + Ok(()) +} + +/// An accepted peer that never speaks again times out: accepting a Connect is +/// the first thing heard from it. +#[test] +fn an_accepted_peer_that_goes_silent_times_out() -> Result<()> { + let mut server = session(500); + server.accept_connect(&session(10).create_connect(0, Vec::new())?, 0, Vec::new())?; + assert!(server.check_timeout(100_000, 1_000)); + Ok(()) +} + +// Ack Cut 1d, batch 2: the connect-attempt timeout. + +/// A server that keeps a session answers a client's repeated or stale Connect +/// with an Ack that may name the client's pending Connect. That Ack says no +/// session started: only an Accept the client honours retires its Connect, so +/// the attempt still times out and is replaced. +#[test] +fn an_ack_naming_the_pending_connect_does_not_retire_it() -> Result<()> { + let mut server = session(500); + let mut old = session(1); + let old_accept = server.accept_connect(&old.create_connect(0, Vec::new())?, 0, Vec::new())?; + old.receive(&old_accept, 0)?; + server.receive(&old.create_ack_for_received(old_accept.sequence), 0)?; + + let mut restarted = session(1); + let first = restarted.create_connect(0, b"join".to_vec())?; + let reply = server.accept_connect(&first, 1, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + assert!(names_sequence(&reply, first.sequence), "the Ack must name the pending Connect for this test to bite"); + restarted.receive(&reply, 1)?; + assert!(!restarted.connected()); + assert_eq!(restarted.pending_reliable_sequences(), vec![first.sequence], "an Ack retired the Connect"); + + let fresh = restarted.due_resends(3_000); + assert_eq!(fresh.len(), 1, "the client waits for ever with nothing to resend"); + assert_eq!(fresh[0].payload, b"join"); + let accept = server.accept_connect(&fresh[0], 3_000, Vec::new())?; + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + restarted.receive(&accept, 3_000)?; + assert!(restarted.connected()); + Ok(()) +} + +/// The fresh attempt is the abandoned sequence plus the receive window less +/// one, so a late copy of the abandoned Connect is stale for the server that +/// took the fresh one: answered with an Ack, no restart, no stranding. +#[test] +fn a_late_copy_of_an_abandoned_connect_is_stale_once_its_replacement_is_accepted() -> Result<()> { + let mut server = session(500); + let mut client = session(10); + let abandoned = client.create_connect(0, Vec::new())?; + let fresh = client.due_resends(3_000); + assert_eq!(fresh.len(), 1); + assert_eq!(fresh[0].sequence, abandoned.sequence + 4_095); + let accept = server.accept_connect(&fresh[0], 3_001, Vec::new())?; + client.receive(&accept, 3_002)?; + server.receive(&client.create_ack_for_received(accept.sequence), 3_002)?; + assert!(client.connected()); + let a = send(&mut client, "a")?; + assert_eq!(names(&server.receive(&a, 3_003)?.delivered), ["a"]); + client.receive(&server.create_ack_for_received(a.sequence), 3_003)?; + + assert!(server.connect_repeats(&abandoned), "the late copy would restart the server"); + let reply = server.accept_connect(&abandoned, 3_004, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + client.receive(&reply, 3_005)?; + assert!(client.connected()); + let b = send(&mut client, "b")?; + assert_eq!(names(&server.receive(&b, 3_006)?.delivered), ["b"]); + Ok(()) +} + +/// A server whose generation sits exactly where the fresh attempt lands takes +/// it for a repeat; the attempt after that leaves the window and is admitted. +#[test] +fn a_server_sitting_at_the_jump_is_left_by_the_next_attempt() -> Result<()> { + let mut server = session(500); + let mut old = session(5_095); + let old_accept = server.accept_connect(&old.create_connect(0, Vec::new())?, 0, Vec::new())?; + old.receive(&old_accept, 0)?; + server.receive(&old.create_ack_for_received(old_accept.sequence), 0)?; + + let mut restarted = session(1_000); + let first = restarted.create_connect(0, b"join".to_vec())?; + restarted.receive(&server.accept_connect(&first, 1, Vec::new())?, 1)?; + assert!(!restarted.connected()); + + let second = restarted.due_resends(3_000).remove(0); + assert_eq!(second.sequence, 5_095, "the jump lands on the server's generation"); + let reply = server.accept_connect(&second, 3_001, Vec::new())?; + assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); + restarted.receive(&reply, 3_001)?; + assert!(!restarted.connected()); + + let third = restarted.due_resends(6_000).remove(0); + assert_eq!(third.sequence, 5_095 + 4_095); + let accept = server.accept_connect(&third, 6_001, Vec::new())?; + assert_eq!(accept.packet_type, CultNetRudpPacketType::Accept); + restarted.receive(&accept, 6_001)?; + assert!(restarted.connected()); + Ok(()) +} + +fn receive_packet(socket: &UdpSocket) -> Result { + let mut buffer = vec![0_u8; 65_535]; + let (received, _) = socket.recv_from(&mut buffer)?; + decode_rudp_packet(&buffer[..received]) +} + +/// A pinned client that restarts on the same address is a repeat by sequence, +/// so the hub answers with an Ack; the client's attempt times out, and the +/// fresh Connect (another sequence) starts a new generation the hub admits. +#[test] +fn the_hub_admits_a_pinned_client_that_restarts_on_the_same_address() -> Result<()> { + let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID))?; + let hub_addr = hub.local_addr()?; + let peer = socket()?; + let mut first = session(1); + peer.send_to(&encode_rudp_packet(&first.create_connect(0, b"join".to_vec())?)?, hub_addr)?; + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))?; + let accept = receive_packet(&peer)?; + first.receive(&accept, 0)?; + peer.send_to(&encode_rudp_packet(&first.create_ack_for_received(accept.sequence))?, hub_addr)?; + hub.receive_event_once()?; + + let mut restarted = session(1); + peer.send_to(&encode_rudp_packet(&restarted.create_connect(0, b"join".to_vec())?)?, hub_addr)?; + hub.receive_event_once()?; + restarted.receive(&receive_packet(&peer)?, 1)?; + assert!(!restarted.connected()); + + let fresh = restarted.due_resends(3_000).remove(0); + peer.send_to(&encode_rudp_packet(&fresh)?, hub_addr)?; + hub_event_matching(&mut hub, |event| matches!(event, CultNetRudpServerEvent::Connected { .. }))?; + restarted.receive(&receive_packet(&peer)?, 3_001)?; + assert!(restarted.connected(), "the restarted pinned client was never admitted"); + Ok(()) +} diff --git a/packages/cultnet-rs/tests/rudp_server_hub.rs b/packages/cultnet-rs/tests/rudp_server_hub.rs index 80cfde3b..ab4ea71c 100644 --- a/packages/cultnet-rs/tests/rudp_server_hub.rs +++ b/packages/cultnet-rs/tests/rudp_server_hub.rs @@ -168,7 +168,7 @@ fn hub_keeps_independent_peer_sessions_and_exposes_connect_evidence() -> Result< } #[test] -fn hub_fences_replaced_generations_and_does_not_replace_connect_retransmits() -> Result<()> { +fn hub_fences_replaced_generations() -> Result<()> { let server_socket = socket()?; let server_addr = server_socket.local_addr()?; let mut hub = CultNetRudpServerHub::new(CultNetRudpServerHubOptions::new( @@ -189,12 +189,6 @@ fn hub_fences_replaced_generations_and_does_not_replace_connect_retransmits() -> })?; let original = connect(&mut hub, &mut provider, &original_evidence)?; - provider.connect(original_evidence.clone())?; - for _ in 0..3 { - assert!(hub.receive_event_once()?.is_none()); - } - assert_eq!(hub.session(original.remote_addr), Some(&original)); - provider.connect(replacement_evidence.clone())?; let mut disconnected = None; for _ in 0..20 { @@ -253,7 +247,7 @@ fn identical_connect_retransmit_reuses_pending_accept_after_loss() -> Result<()> let client_socket = socket()?; let mut client_session = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); @@ -292,13 +286,13 @@ fn identical_connect_retransmit_reuses_pending_accept_after_loss() -> Result<()> fn exact_reliable_acks_clear_more_than_one_ack_window_of_fragments() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); @@ -352,7 +346,7 @@ fn exact_reliable_acks_clear_more_than_one_ack_window_of_fragments() -> Result<( fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); @@ -360,7 +354,7 @@ fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { receiver.set_max_pending_fragment_sets(1)?; let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); @@ -428,13 +422,13 @@ fn session_and_hub_reject_configured_memory_bounds() -> Result<()> { fn replay_history_is_bounded_and_idle_hub_sessions_expire() -> Result<()> { let mut sender = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); let mut receiver = CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id: CONNECTION_ID, - initial_sequence: 1, + initial_sequence: Some(1), resend_delay_ms: 250, max_pending_reliable_packets: None, }); diff --git a/packages/cultnet-rs/tests/rudp_stray_packets.rs b/packages/cultnet-rs/tests/rudp_stray_packets.rs index 33f02794..a64097de 100644 --- a/packages/cultnet-rs/tests/rudp_stray_packets.rs +++ b/packages/cultnet-rs/tests/rudp_stray_packets.rs @@ -30,9 +30,13 @@ fn socket() -> Result { } fn raw_session(connection_id: u32) -> CultNetRudpSession { + raw_session_from(connection_id, 1) +} + +fn raw_session_from(connection_id: u32, initial_sequence: u32) -> CultNetRudpSession { CultNetRudpSession::new(CultNetRudpSessionOptions { connection_id, - initial_sequence: 1, + initial_sequence: Some(initial_sequence), resend_delay_ms: 250, max_pending_reliable_packets: None, }) @@ -442,7 +446,7 @@ fn server_mode_ends_the_session_when_the_session_refuses_a_packet() -> Result<() #[test] fn constructors_reject_limits_that_make_a_connect_unadmittable() -> Result<()> { let mut hub = CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID); - hub.initial_sequence = u32::MAX; + hub.initial_sequence = Some(u32::MAX); assert!(CultNetRudpServerHub::new(hub).is_err()); let mut hub = CultNetRudpServerHubOptions::new("hub", socket()?, CONNECTION_ID); hub.max_pending_reliable_packets = Some(0); @@ -455,13 +459,13 @@ fn constructors_reject_limits_that_make_a_connect_unadmittable() -> Result<()> { ] { let addr = "127.0.0.1:9".parse()?; let mut options = build(socket()?, addr); - options.initial_sequence = u32::MAX; + options.initial_sequence = Some(u32::MAX); assert!(CultNetRudpSocketTransportConnection::new(options).is_err()); let mut options = build(socket()?, addr); options.max_pending_reliable_packets = Some(0); assert!(CultNetRudpSocketTransportConnection::new(options).is_err()); let mut options = build(socket()?, addr); - options.initial_sequence = u32::MAX - 1; + options.initial_sequence = Some(u32::MAX - 1); options.max_pending_reliable_packets = Some(1); assert!(CultNetRudpSocketTransportConnection::new(options).is_ok()); } @@ -725,7 +729,7 @@ fn a_flush_fails_when_a_new_endpoint_replaces_the_peer_mid_wait() -> Result<()> send_to(&peer_a, server_addr, &raw_session(CONNECTION_ID).create_connect(0, b"a".to_vec())?)?; let _ = server.receive_once()?; server.send_reliable("schema", b"for A".to_vec())?; - send_to(&peer_b, server_addr, &raw_session(CONNECTION_ID).create_connect(0, b"b".to_vec())?)?; + send_to(&peer_b, server_addr, &raw_session_from(CONNECTION_ID, 2).create_connect(0, b"b".to_vec())?)?; let error = server.flush_reliable(Duration::from_millis(300)).expect_err("A's write was forgotten"); assert!( @@ -745,8 +749,224 @@ fn a_repeated_connect_acknowledges_what_it_carries() -> Result<()> { let accept = server.accept_connect(&connect, 0, Vec::new())?; let mut repeat = connect.clone(); repeat.ack = accept.sequence; - let reply = server.answer_repeated_connect(&repeat, 1)?; + let reply = server.accept_connect(&repeat, 1, Vec::new())?; assert_eq!(reply.packet_type, CultNetRudpPacketType::Ack); assert_eq!(server.outstanding_reliable_packet_count(), 0); Ok(()) } + +/// A connected client session with one ordered write the peer never received. +fn client_with_a_lost_write() -> Result<(CultNetRudpSession, CultNetRudpSession)> { + let mut client = raw_session(CONNECTION_ID); + let mut server = raw_session(CONNECTION_ID); + let accept = server.accept_connect(&client.create_connect(0, Vec::new())?, 0, Vec::new())?; + client.receive(&accept, 0)?; + client.send( + "schema", + b"owed to the old session".to_vec(), + CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..Default::default() + }, + )?; + // A fragmented write larger than the send window leaves part of it queued. + client.send_many( + "schema", + vec![7u8; 40 * 8], + CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..Default::default() + }, + Some(8), + )?; + assert_eq!(client.outstanding_reliable_packet_count(), 41); + assert!(client.queued_reliable_packet_count() > 0); + Ok((client, server)) +} + +/// A write belongs to the session it was issued in. Whichever way that session +/// ends, the write it still owed dies with it: it is not retransmitted into a +/// later session, where the new peer would deliver it. +#[test] +fn every_way_a_session_ends_drops_the_writes_it_owed() -> Result<()> { + type End = fn(&mut CultNetRudpSession, &mut CultNetRudpSession) -> Result<()>; + let endings: [(&str, End); 4] = [ + ("peer Disconnect", |client, server| { + client.receive(&server.create_disconnect(b"bye".to_vec()), 1)?; + Ok(()) + }), + ("local disconnect", |client, _| { + client.create_disconnect(b"bye".to_vec()); + Ok(()) + }), + ("timeout", |client, _| { + assert!(client.check_timeout(1_000, 10)); + Ok(()) + }), + ("refusal", |client, _| { + client.end_refused_session(); + Ok(()) + }), + ]; + for (name, end) in endings { + let (mut client, mut server) = client_with_a_lost_write()?; + end(&mut client, &mut server)?; + assert_eq!(client.outstanding_reliable_packet_count(), 0, "{name}: the write survived the end"); + + let mut next_server = raw_session(CONNECTION_ID); + let connect = client.create_connect(2_000, Vec::new())?; + next_server.accept_connect(&connect, 2_000, Vec::new())?; + for resend in client.due_resends(60_000) { + let result = next_server.receive(&resend, 60_000)?; + assert!( + result.delivered.is_empty(), + "{name}: the old session's write was delivered in the next session" + ); + } + } + Ok(()) +} + +fn connected_client(server_socket: &UdpSocket, server_addr: SocketAddr) -> Result<(CultNetRudpSocketTransportConnection, CultNetRudpSession, SocketAddr)> { + let mut client = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::client("client", socket()?, server_addr, CONNECTION_ID), + )?; + client.connect(b"hello".to_vec())?; + let mut buffer = vec![0_u8; 65_535]; + let (received, client_addr) = server_socket.recv_from(&mut buffer)?; + let mut peer = raw_session(CONNECTION_ID); + let accept = peer.accept_connect(&decode_rudp_packet(&buffer[..received])?, 0, Vec::new())?; + send_to(server_socket, client_addr, &accept)?; + let _ = client.receive_once()?; + Ok((client, peer, client_addr)) +} + +/// A local `disconnect()` ends the session like any other end: the receipt of a +/// write the peer never acknowledged is Invalidated, stays so after a +/// reconnect, and the write is not carried into the new session. +#[test] +fn a_receipt_invalidated_by_a_local_disconnect_stays_invalidated_after_reconnecting() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let (mut client, _peer, _) = connected_client(&server, server_addr)?; + let receipt = client.send_reliable("schema", b"written before the disconnect".to_vec())?; + assert_eq!(client.reliable_send_status(&receipt), CultNetRudpReliableSendStatus::Pending); + + client.disconnect(b"bye".to_vec())?; + assert_eq!(client.reliable_send_status(&receipt), CultNetRudpReliableSendStatus::Invalidated); + assert_eq!(client.outstanding_reliable_packet_count(), 0, "the write survived the disconnect"); + + client.connect(b"hello again".to_vec())?; + assert_eq!( + client.reliable_send_status(&receipt), + CultNetRudpReliableSendStatus::Invalidated, + "reconnecting brought the write back" + ); + assert_eq!(client.outstanding_reliable_packet_count(), 1, "only the new Connect is owed"); + Ok(()) +} + +/// A flush that starts after the session ended has no live session to wait on. +#[test] +fn a_flush_fails_after_a_local_disconnect_or_a_timeout() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let (mut client, _peer, _) = connected_client(&server, server_addr)?; + client.send_reliable("schema", b"owed".to_vec())?; + client.disconnect(b"bye".to_vec())?; + let error = client.flush_reliable(Duration::from_millis(200)).expect_err("the write was dropped"); + assert!(error.to_string().contains("ended before its reliable writes were acknowledged"), "{error}"); + + let server = socket()?; + let server_addr = server.local_addr()?; + let (mut client, _peer, _) = connected_client(&server, server_addr)?; + client.send_reliable("schema", b"owed".to_vec())?; + std::thread::sleep(Duration::from_millis(5)); + assert!(client.check_timeout(1)); + let error = client.flush_reliable(Duration::from_millis(200)).expect_err("the write was dropped"); + assert!(error.to_string().contains("ended before its reliable writes were acknowledged"), "{error}"); + Ok(()) +} + +/// A flush that starts after a reconnect waits on the new session: the end of +/// the old one does not fail it. +#[test] +fn a_flush_started_after_a_reconnect_waits_on_the_new_session() -> Result<()> { + let server = socket()?; + let server_addr = server.local_addr()?; + let (mut client, _peer, client_addr) = connected_client(&server, server_addr)?; + client.send_reliable("schema", b"owed to the old session".to_vec())?; + client.disconnect(b"bye".to_vec())?; + client.connect(b"hello again".to_vec())?; + let connect = drain(&server)? + .into_iter() + .rfind(|packet| packet.packet_type == CultNetRudpPacketType::Connect) + .expect("the reconnect reached the peer"); + let accept = raw_session(CONNECTION_ID).accept_connect(&connect, 0, Vec::new())?; + send_to(&server, client_addr, &accept)?; + let _ = client.receive_once()?; + + client.flush_reliable(Duration::from_millis(500))?; + Ok(()) +} + +/// A server-mode transport that accepts a new Connect after its peer left has a +/// live session again, and a flush started in it waits on that session. +#[test] +fn a_flush_started_after_the_server_accepts_a_new_connect_waits_on_the_new_session() -> Result<()> { + let server_socket = socket()?; + let server_addr = server_socket.local_addr()?; + let mut server = CultNetRudpSocketTransportConnection::new( + CultNetRudpSocketTransportOptions::server("server", server_socket, CONNECTION_ID), + )?; + let peer = socket()?; + let mut first = raw_session(CONNECTION_ID); + send_to(&peer, server_addr, &first.create_connect(0, b"peer".to_vec())?)?; + let _ = server.receive_once()?; + send_to(&peer, server_addr, &first.create_disconnect(b"bye".to_vec()))?; + let _ = server.receive_once()?; + + let mut next = raw_session(CONNECTION_ID); + send_to(&peer, server_addr, &next.create_connect(0, b"peer".to_vec())?)?; + let _ = server.receive_once()?; + let accept = drain(&peer)? + .into_iter() + .rfind(|packet| packet.packet_type == CultNetRudpPacketType::Accept) + .expect("the new Connect was accepted"); + next.receive(&accept, 0)?; + send_to(&peer, server_addr, &next.create_ack_for_received(accept.sequence))?; + let _ = server.receive_once()?; + + server.flush_reliable(Duration::from_millis(500))?; + Ok(()) +} + +/// The session ended, the peer did not notice: what it sent before the end is +/// still remembered, held frames included, so its retransmits are duplicates. +#[test] +fn an_ending_does_not_forget_what_was_received_from_the_peer() -> Result<()> { + let mut client = raw_session(CONNECTION_ID); + let mut server = raw_session(CONNECTION_ID); + let accept = server.accept_connect(&client.create_connect(0, Vec::new())?, 0, Vec::new())?; + client.receive(&accept, 0)?; + let ordered = CultNetRudpSendOptions { + reliable: true, + ordered: true, + ..Default::default() + }; + let s1 = client.send("schema", b"s1".to_vec(), ordered.clone())?; + let s2 = client.send("schema", b"s2".to_vec(), ordered)?; + assert!(server.receive(&s2, 1)?.delivered.is_empty(), "s1 is missing, so s2 is held"); + + assert!(server.check_timeout(1_000, 10)); + let released = server.receive(&s1, 1_001)?.delivered; + assert_eq!( + released.iter().map(|frame| frame.payload.clone()).collect::>(), + [b"s1".to_vec(), b"s2".to_vec()], + "the ending forgot the held frame" + ); + assert!(server.receive(&s2, 1_002)?.delivered.is_empty(), "the ending forgot what was received"); + Ok(()) +} diff --git a/packages/cultnet-ts/src/rudp.ts b/packages/cultnet-ts/src/rudp.ts index ae7ea838..e32bb2d7 100644 --- a/packages/cultnet-ts/src/rudp.ts +++ b/packages/cultnet-ts/src/rudp.ts @@ -1,3 +1,4 @@ +import { randomInt } from "node:crypto"; import { EventEmitter } from "node:events"; import { type RemoteInfo, type Socket } from "node:dgram"; @@ -18,6 +19,34 @@ const RUDP_FIXED_HEADER_BYTES = 36; const MAX_CHANNEL_ID_BYTES = 255; export const CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS = 32; const RUDP_RECEIVED_SEQUENCE_WINDOW = 4_096; +/// How long a client keeps retransmitting a Connect nobody answered before it +/// abandons that attempt for a fresh one. A Connect the server answers with an +/// Ack, never an Accept, is one the server judged stale (see `acceptConnect`); +/// retransmitting the same sequence would never change its mind. +const RUDP_CONNECT_ATTEMPT_MS = 3_000; + +/// A sequence drawn from a secure source in [1, 2^31). The Connect's sequence +/// is what tells a peer whether a Connect repeats one it already accepted or +/// starts a new session, so two sessions must not share one by default. +function drawSequence(): number { + return randomInt(1, 2 ** 31); +} + +/// Whether `sequence` is at or before `mark` in serial order, within the +/// receive window. The compare is modular, so it holds across the wrap of the +/// 32-bit space; a sequence further back than the window is the duplicate +/// test's below-window clause, and one ahead of the mark is never before it. +function atOrBefore(sequence: number, mark: number): boolean { + return ((mark - sequence) >>> 0) < RUDP_RECEIVED_SEQUENCE_WINDOW; +} + +function packetAcknowledges(packet: CultNetRudpPacket, sequence: number): boolean { + if (packet.ack === sequence) return true; + for (let bit = 0; bit < 32; bit += 1) { + if ((packet.ackMask & (1 << bit)) !== 0 && packet.ack - bit - 1 === sequence) return true; + } + return false; +} export type CultNetRudpPacketType = | "connect" @@ -72,11 +101,6 @@ type PendingReliablePacket = { lastSentAtMs: number; }; -type PendingOrderedFrame = { - frame: CultNetRudpDeliveredFrame; - nextSequence: number; -}; - type FragmentBuffer = { channelId: string; reliable: boolean; @@ -150,6 +174,24 @@ export class CultNetRudpSession { readonly #nextSequencedByChannel = new Map(); #nextFragmentId = 1; #connected = false; + /// Advances every time a generation ends. Everything issued in a generation + /// (writes, flushes) belongs to it and dies with it. + #generation = 0; + /// True from the end of a generation until the next Connect or Accept begins + /// one. A flush started in that interval has no live generation to wait on. + #ended = false; + /// The sequence of the Connect that started the current generation: sent by + /// this side, or accepted from the peer. A Connect repeats exactly when the + /// session is connected and the Connect carries this sequence. + #connectSequence: number | undefined; + /// A Connect this side sent is unanswered. Only then is an Accept honoured. + #awaitingAccept = false; + /// The payload of the Connect this side sent, kept so a fresh attempt can + /// always be built whatever became of the pending packet. + #connectPayload = new Uint8Array(); + /// When the unanswered Connect first went out; `dueResends` abandons it for a + /// fresh attempt once `RUDP_CONNECT_ATTEMPT_MS` has passed. + #connectStartedAtMs = 0; readonly #maxPendingReliablePackets: number | undefined; #lastReceivedAtMs: number | undefined; #highestReceivedSequence: number | undefined; @@ -157,8 +199,15 @@ export class CultNetRudpSession { readonly #latestSequencedByChannel = new Map(); readonly #pendingReliable = new Map(); readonly #queuedReliable: CultNetRudpPacket[] = []; - readonly #orderedNextSequenceByChannel = new Map(); - readonly #orderedBuffers = new Map>(); + /// Every reliable sequence up to and including this one has been received + /// since the peer state was last reset. Only the handshake seeds it: the + /// peer's Connect on the accepting side, the peer's Accept on the connecting + /// side. Reliable data that arrives before that is refused. + #receivedThrough: number | undefined; + /// Ordered frames received but not yet deliverable, keyed by first sequence. + /// A frame is held for exactly one reason: a reliable sequence below it has + /// not arrived. + readonly #orderedHeld = new Map(); readonly #fragmentBuffers = new Map(); /// Matches cultnet-rs's `max_pending_fragment_sets`. A fragment set is only /// removed on successful reassembly, so a set that loses one fragment is @@ -169,7 +218,7 @@ export class CultNetRudpSession { constructor(options: CultNetRudpSessionOptions) { this.connectionId = toUint32(options.connectionId, "connectionId"); - this.#nextSequence = toUint32(options.initialSequence ?? 1, "initialSequence"); + this.#nextSequence = toUint32(options.initialSequence ?? drawSequence(), "initialSequence"); if (this.#nextSequence === 0xffff_ffff) { throw new Error("RUDP initialSequence must leave room for a reliable packet."); } @@ -211,23 +260,53 @@ export class CultNetRudpSession { return this.#lastReceivedAtMs; } - resetPeerState(): void { + get generation(): number { + return this.#generation; + } + + get ended(): boolean { + return this.#ended; + } + + /// The one way a session generation ends. The session stops being connected + /// and what it still owed the peer dies with it: a write not yet acknowledged + /// is dropped, so no later session retransmits it or credits an ack to it. + /// What was learned from the peer is not touched: the peer may not know the + /// session ended, and forgetting what it sent would let its retransmits be + /// delivered twice. + #endSession(): void { this.#connected = false; + this.#ended = true; + this.#awaitingAccept = false; + this.#generation += 1; + this.#pendingReliable.clear(); + this.#queuedReliable.splice(0); + } + + /// Ends the current generation and forgets everything learned from the peer; + /// sequence numbers already issued stay issued. + resetPeerState(): void { + this.#endSession(); this.#lastReceivedAtMs = undefined; this.#highestReceivedSequence = undefined; this.#receivedSequences.clear(); this.#nextSequencedByChannel.clear(); this.#latestSequencedByChannel.clear(); - this.#pendingReliable.clear(); - this.#queuedReliable.splice(0); - this.#orderedNextSequenceByChannel.clear(); - this.#orderedBuffers.clear(); + this.#receivedThrough = undefined; + this.#orderedHeld.clear(); this.#fragmentBuffers.clear(); this.#fragmentSetsEvicted = 0; } createConnect(nowMs = 0, payload = new Uint8Array()): CultNetRudpPacket { + // A session that has had a peer starts a new generation: nothing it learned + // from that peer describes the one this Connect reaches, and what it still + // owed that peer no longer takes room in the queue. + if (this.#connectSequence !== undefined) { + this.resetPeerState(); + } this.#ensureReliableCapacity(1); + this.#ended = false; const packet = this.#createPacket({ packetType: "connect", channelId: "control", @@ -235,24 +314,68 @@ export class CultNetRudpSession { ordered: true, payload, }); + this.#connectSequence = packet.sequence; + this.#awaitingAccept = true; + this.#connectStartedAtMs = nowMs; + this.#connectPayload = new Uint8Array(payload); this.#trackReliable(packet, nowMs); return packet; } + /// Whether `packet` is a Connect this session's current generation already + /// owns: a retransmit of the Connect that started it (the session is + /// connected and the sequence is that Connect's), or a stale copy of an + /// earlier attempt by the same client (a sequence before it, within the + /// receive window). Servers that keep one session per peer ask this to tell a + /// Connect that starts a new session from one that does not; anything else + /// that reaches `acceptConnect` starts a new generation. + connectRepeats(packet: CultNetRudpPacket): boolean { + return packet.packetType === "connect" + && this.#connected + && this.#connectSequence !== undefined + && (this.#connectSequence === packet.sequence + || this.#connectIsStale(packet.sequence, this.#connectSequence)); + } + + /// A Connect that precedes the current generation's within the receive window + /// is the client's earlier attempt, delayed in the network: a client that + /// retried never sends a lower sequence again. Restarting on it would strand + /// the client, which honours only the Accept for its newest Connect. The rule + /// is TCP's answer to a delayed SYN (RFC 5961's challenge ACK): keep the + /// connection and answer with an Ack. A restarted client whose random initial + /// sequence lands in this window is answered the same way and abandons the + /// attempt for a fresh draw (`RUDP_CONNECT_ATTEMPT_MS`). + #connectIsStale(sequence: number, current: number): boolean { + return sequence !== current && atOrBefore(sequence, current); + } + + /// Answers a Connect. A repeat of the accepted one queues nothing, so a + /// Connect storm cannot grow the reliable queue: the reply is the Accept + /// still awaiting acknowledgement, or an Ack once it was acknowledged. A + /// stale copy of an earlier attempt gets the same reply and changes nothing + /// else: it is not evidence the peer is alive. Any other Connect ends the + /// current generation, forgets the peer and accepts a new one. acceptConnect(packet: CultNetRudpPacket, nowMs = 0, payload = new Uint8Array()): CultNetRudpPacket { this.#requireConnection(packet); if (packet.packetType !== "connect") { throw new Error(`Expected RUDP connect packet, got ${packet.packetType}.`); } - // A Connect from a peer this session already accepted repeats. - if (this.#connected) { - return this.answerRepeatedConnect(packet, nowMs); + if (this.connectRepeats(packet)) { + if (this.#connectSequence === packet.sequence) { + this.#applyAcknowledgements(packet); + this.#rememberReceived(packet.sequence); + this.#lastReceivedAtMs = nowMs; + } + return this.#pendingAcceptForResend(nowMs) ?? this.createAck(); } + this.resetPeerState(); this.#ensureReliableCapacity(1); - this.#rememberReceived(packet.sequence); + this.#seedReceived(packet.sequence); this.#lastReceivedAtMs = nowMs; + this.#connectSequence = packet.sequence; this.#connected = true; + this.#ended = false; const response = this.#createPacket({ packetType: "accept", channelId: "control", @@ -264,24 +387,12 @@ export class CultNetRudpSession { return response; } - /** - * Answers a Connect from a peer this session has already accepted. The packet's - * sequence is remembered and nothing is queued, so a Connect storm cannot grow - * the reliable queue. The reply is the Accept still awaiting acknowledgement, - * or an Ack once it was acknowledged. - */ - answerRepeatedConnect(packet: CultNetRudpPacket, nowMs = 0): CultNetRudpPacket { - this.#requireConnection(packet); - if (packet.packetType !== "connect") { - throw new Error(`Expected RUDP connect packet, got ${packet.packetType}.`); - } - this.#applyAcknowledgements(packet); - this.#rememberReceived(packet.sequence); - this.#lastReceivedAtMs = nowMs; + /// The Accept still awaiting acknowledgement, resent. + #pendingAcceptForResend(nowMs: number): CultNetRudpPacket | undefined { const pendingAccept = [...this.#pendingReliable.values()] .find(pending => pending.packet.packetType === "accept"); if (!pendingAccept) { - return this.createAck(); + return undefined; } pendingAccept.lastSentAtMs = nowMs; return { @@ -369,15 +480,28 @@ export class CultNetRudpSession { receive(packet: CultNetRudpPacket, nowMs = 0): CultNetRudpReceiveResult { this.#requireConnection(packet); - this.#applyAcknowledgements(packet); + // An Accept counts only while this side's Connect is unanswered and the + // Accept names it. A late or duplicate one, or one from an earlier + // generation, must not seed the watermark or revive an ended session. + const honoursAccept = packet.packetType === "accept" + && this.#awaitingAccept + && this.#connectSequence !== undefined + && packetAcknowledges(packet, this.#connectSequence); + if (packet.packetType === "accept" && !honoursAccept) { + return { delivered: [], readyToSend: [] }; + } + // While this side's Connect awaits its Accept, only the Accept it honours + // retires it: an Ack that names the Connect (a server's reply to a repeat or + // a stale copy) says the server did not start a session. + if (honoursAccept || !this.#awaitingAccept) { + this.#applyAcknowledgements(packet); + } const readyToSend = this.#promoteQueuedReliable(nowMs); this.#lastReceivedAtMs = nowMs; - const expectedSequenceIfUninitialized = this.#highestReceivedSequence === undefined - ? packet.sequence - : this.#highestReceivedSequence + 1; - if (packet.packetType === "accept") { - this.#rememberReceived(packet.sequence); + if (honoursAccept) { + this.#awaitingAccept = false; + this.#seedReceived(packet.sequence); this.#connected = true; return { delivered: [], readyToSend }; } @@ -404,7 +528,7 @@ export class CultNetRudpSession { } if (packet.packetType === "disconnect") { - this.#connected = false; + this.#endSession(); return { delivered: [], readyToSend, @@ -417,39 +541,41 @@ export class CultNetRudpSession { return { delivered: [], readyToSend }; } - const isDuplicate = (packet.reliable ?? false) - && (this.#receivedSequences.has(packet.sequence) - || (this.#highestReceivedSequence !== undefined - && packet.sequence < this.#highestReceivedSequence - && this.#highestReceivedSequence - packet.sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW)); + // Reliable data before the handshake has seeded the watermark has no place + // in the order: refuse it unremembered, so it is not acknowledged and the + // sender retransmits it once the handshake is done. + if ((packet.reliable ?? false) && this.#receivedThrough === undefined) { + return { delivered: [], readyToSend }; + } + + const isDuplicate = (packet.reliable ?? false) && this.#wasReceived(packet.sequence); if (packet.reliable ?? false) this.#rememberReceived(packet.sequence); if (isDuplicate) { return { delivered: [], readyToSend }; } + const delivered: CultNetRudpDeliveredFrame[] = []; const reassembled = this.#reassemble(packet); - if (!reassembled) { - return { delivered: [], readyToSend }; - } - - if (!reassembled.ordered && (packet.sequenced ?? false)) { - const newestSequence = reassembled.nextSequence - 1; - const latestSequence = this.#latestSequencedByChannel.get(reassembled.frame.channelId); - if (latestSequence !== undefined && newestSequence <= latestSequence) { - return { delivered: [], readyToSend }; + if (reassembled) { + if (reassembled.ordered) { + this.#orderedHeld.set(reassembled.frame.sequence, reassembled.frame); + } else if (!(packet.sequenced ?? false)) { + delivered.push(reassembled.frame); + } else { + const newestSequence = reassembled.nextSequence - 1; + const latestSequence = this.#latestSequencedByChannel.get(reassembled.frame.channelId); + if (latestSequence === undefined || newestSequence > latestSequence) { + this.#latestSequencedByChannel.set(reassembled.frame.channelId, newestSequence); + delivered.push(reassembled.frame); + } } - this.#latestSequencedByChannel.set(reassembled.frame.channelId, newestSequence); - return { delivered: [reassembled.frame], readyToSend }; } - if (!reassembled.ordered) { - return { delivered: [reassembled.frame], readyToSend }; - } - - return { - delivered: this.#deliverOrdered(reassembled.frame, reassembled.nextSequence, expectedSequenceIfUninitialized), - readyToSend, - }; + // Any reliable packet may have advanced the watermark, a fragment or an + // unordered frame as much as an ordered one, so the drain runs after all of + // them. + delivered.push(...this.#drainOrdered()); + return { delivered, readyToSend }; } createAck(): CultNetRudpPacket { @@ -490,7 +616,12 @@ export class CultNetRudpSession { } createAckForReceived(sequence: number): CultNetRudpPacket { + // What `receive` refused is not acknowledged by name (the ack carries only + // what was received), so its sender retransmits it. const receivedSequence = toUint32(sequence, "received sequence"); + if (!this.#wasReceived(receivedSequence)) { + return this.createAck(); + } const { ack } = this.#ackState(); return ack >= receivedSequence && ack - receivedSequence <= 32 ? this.createAck() @@ -506,7 +637,7 @@ export class CultNetRudpSession { } createDisconnect(reason = new Uint8Array()): CultNetRudpPacket { - this.#connected = false; + this.#endSession(); return this.#createPacket({ packetType: "disconnect", channelId: "control", @@ -521,11 +652,15 @@ export class CultNetRudpSession { if (nowMs - this.#lastReceivedAtMs <= timeoutMs) { return false; } - this.#connected = false; + this.#endSession(); return true; } dueResends(nowMs: number): CultNetRudpPacket[] { + const fresh = this.#abandonUnansweredConnect(nowMs); + if (fresh) { + return [fresh]; + } const due: CultNetRudpPacket[] = []; for (const pending of this.#pendingReliable.values()) { if (nowMs - pending.lastSentAtMs >= this.resendDelayMs) { @@ -536,6 +671,27 @@ export class CultNetRudpSession { return due.sort((left, right) => left.sequence - right.sequence); } + /// A Connect unanswered for `RUDP_CONNECT_ATTEMPT_MS` is replaced, not + /// retransmitted further, by a Connect with the same payload and the + /// abandoned sequence plus the receive window less one. Nothing was ever sent + /// in an unanswered generation, so no sequence issued so far is owed. The jump + /// keeps a late copy of the abandoned Connect inside the new one's stale + /// window, so a server that took the new Connect answers the copy with an Ack + /// instead of restarting; and it leaves the stale window of whatever + /// generation the server holds, unless that generation sits exactly at the + /// jump, and then the next attempt leaves it. The first Connect of a session + /// is the only one drawn at random. + #abandonUnansweredConnect(nowMs: number): CultNetRudpPacket | undefined { + if (!this.#awaitingAccept || nowMs - this.#connectStartedAtMs < RUDP_CONNECT_ATTEMPT_MS) { + return undefined; + } + if (this.#connectSequence === undefined) { + return undefined; + } + this.#nextSequence = Math.min(this.#connectSequence + RUDP_RECEIVED_SEQUENCE_WINDOW - 1, 0xffff_fffe); + return this.createConnect(nowMs, this.#connectPayload); + } + #createPacket(packet: { packetType: CultNetRudpPacketType; channelId: string; @@ -616,8 +772,34 @@ export class CultNetRudpSession { } } + /// The handshake's one act on the watermark: the peer's Connect or Accept is + /// the first sequence of the session, whatever else it has sent. + #seedReceived(sequence: number): void { + this.#receivedThrough = sequence; + this.#rememberReceived(sequence); + } + + /// The one duplicate test: true for a sequence in the window that was + /// received, for one below the window, and for one at or before the + /// watermark. The watermark starts at the handshake's seed, so a frame the + /// peer sent in an earlier generation (every sequence it issued is below the + /// Connect that began this one) is a duplicate of something already delivered + /// and is acknowledged, never delivered again. + #wasReceived(sequence: number): boolean { + return this.#receivedSequences.has(sequence) + || (this.#highestReceivedSequence !== undefined + && sequence < this.#highestReceivedSequence + && this.#highestReceivedSequence - sequence >= RUDP_RECEIVED_SEQUENCE_WINDOW) + || (this.#receivedThrough !== undefined && atOrBefore(sequence, this.#receivedThrough)); + } + #rememberReceived(sequence: number): void { this.#receivedSequences.add(sequence); + if (this.#receivedThrough !== undefined) { + let through = this.#receivedThrough; + while (this.#receivedSequences.has(through + 1)) through += 1; + this.#receivedThrough = through; + } if (this.#highestReceivedSequence === undefined || sequence > this.#highestReceivedSequence) { this.#highestReceivedSequence = sequence; } @@ -734,68 +916,20 @@ export class CultNetRudpSession { }; } - #deliverOrdered( - frame: CultNetRudpDeliveredFrame, - nextSequence: number, - expectedSequenceIfUninitialized: number, - ): CultNetRudpDeliveredFrame[] { - let next = this.#orderedNextSequenceByChannel.get(frame.channelId); - if (next === undefined) { - next = Math.min(expectedSequenceIfUninitialized, frame.sequence); - this.#orderedNextSequenceByChannel.set(frame.channelId, next); - } - - while (frame.sequence > next && this.#receivedSequences.has(next) && !this.#orderedBuffers.get(frame.channelId)?.has(next)) { - next += 1; - this.#orderedNextSequenceByChannel.set(frame.channelId, next); - } - - if (frame.sequence < next) { - return []; - } - - if (frame.sequence > next) { - let buffer = this.#orderedBuffers.get(frame.channelId); - if (!buffer) { - buffer = new Map(); - this.#orderedBuffers.set(frame.channelId, buffer); - } - buffer.set(frame.sequence, { frame, nextSequence }); - return []; - } - - this.#orderedNextSequenceByChannel.set(frame.channelId, nextSequence); - return [ - frame, - ...this.#drainOrdered(frame.channelId), - ]; - } - - #drainOrdered(channelId: string): CultNetRudpDeliveredFrame[] { - const delivered: CultNetRudpDeliveredFrame[] = []; - const buffer = this.#orderedBuffers.get(channelId); - if (!buffer) { - return delivered; - } - - let next = this.#orderedNextSequenceByChannel.get(channelId); - while (next !== undefined && buffer.has(next)) { - const pending = buffer.get(next)!; - buffer.delete(next); - delivered.push(pending.frame); - next = pending.nextSequence; - this.#orderedNextSequenceByChannel.set(channelId, next); - this.#skipReceivedNonChannelSequences(channelId); - } - return delivered; - } - - #skipReceivedNonChannelSequences(channelId: string): void { - let next = this.#orderedNextSequenceByChannel.get(channelId); - while (next !== undefined && this.#receivedSequences.has(next) && !this.#orderedBuffers.get(channelId)?.has(next)) { - next += 1; - this.#orderedNextSequenceByChannel.set(channelId, next); - } + /// Delivers, in sequence order, every held ordered frame whose first sequence + /// is at most one past the watermark. Ordered delivery has one owner: the + /// watermark. A frame this call does not deliver waits for the sequence below + /// it, and nothing else releases it. + #drainOrdered(): CultNetRudpDeliveredFrame[] { + if (this.#receivedThrough === undefined) return []; + const releasable = [...this.#orderedHeld.keys()] + .filter((sequence) => sequence <= this.#receivedThrough! + 1) + .sort((left, right) => left - right); + return releasable.map((sequence) => { + const frame = this.#orderedHeld.get(sequence)!; + this.#orderedHeld.delete(sequence); + return frame; + }); } #allocateFragmentId(): number { @@ -824,12 +958,8 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement #remoteHost: string | undefined; #remotePort: number | undefined; #closed = false; - // A flush belongs to the session it started in. Each session start advances - // the generation; each session end records the generation it ended and why. - // A reconnect from a disconnect listener starts a new generation but cannot - // un-end the old one. - #generation = 0; - #endedGeneration = -1; + // Why the last generation ended. A report only: the session owns whether it + // ended. #endedReason: Uint8Array | undefined; readonly #stats: CultNetTransportStats = { bytesReceived: 0, @@ -892,7 +1022,6 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement if (this.#mode !== "client") { throw new Error("Only a client RUDP socket transport can initiate connect."); } - this.#generation += 1; this.#sendPacket(this.#session.createConnect(Date.now(), payload)); } @@ -910,11 +1039,14 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement async flush(timeoutMs = 30_000): Promise { const deadline = Date.now() + Math.max(0, timeoutMs); - const generation = this.#generation; + // A flush belongs to the generation it started in: whatever ends that + // generation, and whatever begins after it (a reconnect from a disconnect + // listener included), the writes being waited on are gone. + const generation = this.#session.generation; for (;;) { // An ended session forgot its unacknowledged writes; reporting them - // flushed would be a lie, even if a new session has since begun. - if (this.#endedGeneration >= generation) { + // flushed would be a lie. + if (this.#session.ended || this.#session.generation !== generation) { throw new Error( `RUDP session ended before its reliable writes were acknowledged: ${Buffer.from(this.#endedReason ?? new Uint8Array()).toString("utf8")}`, ); @@ -934,11 +1066,6 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } } - #endGeneration(reason: Uint8Array): void { - this.#endedGeneration = this.#generation; - this.#endedReason = reason; - } - ping(payload = new Uint8Array()): void { this.#sendPacket(this.#session.createPing(payload)); } @@ -946,7 +1073,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement checkTimeout(timeoutMs: number, nowMs = Date.now()): boolean { const timedOut = this.#session.checkTimeout(nowMs, timeoutMs); if (timedOut) { - this.#endGeneration(Buffer.from("session timed out", "utf8")); + this.#endedReason = Buffer.from("session timed out", "utf8"); this.emit("timeout"); this.emit("close"); } @@ -973,7 +1100,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement // The goodbye is built after the reset, or its ack field would acknowledge // the very frame the session refused. this.#session.resetPeerState(); - this.#endGeneration(reason); + this.#endedReason = reason; try { this.#sendPacket(this.#session.createDisconnect(reason)); } catch { @@ -1008,7 +1135,10 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement return; } - let movedEndpoint = false; + // A Connect from another endpoint is a new client, whatever sequence it + // carries: the endpoint moves and the Connect starts a new generation. Only + // a Connect from the peer's own endpoint can be its repeat. + let connectFromNewEndpoint = false; if (!this.#remoteHost || this.#remotePort === undefined) { if (this.#mode === "server" && packet.packetType !== "connect") { this.#stats.packetsDropped += 1; @@ -1018,9 +1148,9 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement this.#remotePort = remote.port; } else if (remote.address !== this.#remoteHost || remote.port !== this.#remotePort) { if (this.#mode === "server" && packet.packetType === "connect") { - movedEndpoint = true; this.#remoteHost = remote.address; this.#remotePort = remote.port; + connectFromNewEndpoint = true; } else { this.#stats.packetsDropped += 1; return; @@ -1028,22 +1158,19 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } if (this.#mode === "server" && packet.packetType === "connect") { - // Only a Connect from the endpoint already accepted repeats, and the - // session answers it with the Accept owed. A Connect from another - // endpoint replaces the peer: the old session's writes die with it. - const repeated = this.#session.connected && !movedEndpoint; - if (!repeated) { - // A timed-out session already ended its generation; any other - // predecessor ends here, connected or not. - if (this.#endedGeneration < this.#generation) { - this.#endGeneration(Buffer.from("replaced by a new Connect", "utf8")); - } + // The session decides whether this Connect repeats the one it accepted; a + // repeat is answered with the Accept owed, anything else replaces the + // peer and the old session's writes die with it. A timed-out session + // already ended its generation; any other predecessor ends here. + if ((connectFromNewEndpoint || !this.#session.connectRepeats(packet)) && !this.#session.ended) { + this.#endedReason = Buffer.from("replaced by a new Connect", "utf8"); + } + if (connectFromNewEndpoint) { this.#session.resetPeerState(); } let accept: CultNetRudpPacket; try { accept = this.#session.acceptConnect(packet, Date.now()); - if (!repeated) this.#generation += 1; } catch { this.#stats.packetsDropped += 1; return; @@ -1081,7 +1208,7 @@ export class CultNetRudpSocketTransportConnection extends EventEmitter implement } satisfies CultNetTransportFrame); } if (result.disconnected) { - this.#endGeneration(result.disconnectReason ?? new Uint8Array()); + this.#endedReason = result.disconnectReason ?? new Uint8Array(); this.emit("disconnect", { reason: result.disconnectReason ?? new Uint8Array() }); this.emit("close"); return; diff --git a/packages/cultnet-ts/test/cultnet.test.ts b/packages/cultnet-ts/test/cultnet.test.ts index 35cba767..4a0c3bbc 100644 --- a/packages/cultnet-ts/test/cultnet.test.ts +++ b/packages/cultnet-ts/test/cultnet.test.ts @@ -8,7 +8,7 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; import { z } from "zod"; -import { encode } from "@msgpack/msgpack"; +import { decode as decodeMsgpack, encode } from "@msgpack/msgpack"; import { CultCache, SingleFileMessagePackBackingStore, @@ -166,6 +166,14 @@ function createDuplexPair(): { a: Duplex; b: Duplex } { return { a, b }; } +/// Whether the packet's ack field and mask cover the sequence. +function acknowledges(packet: CultNetRudpPacket, sequence: number): boolean { + if (packet.ack === sequence) return true; + return packet.ack > sequence + && packet.ack - sequence <= 32 + && (packet.ackMask & (1 << (packet.ack - sequence - 1))) !== 0; +} + async function bindUdpSocket(): Promise { const socket = dgram.createSocket("udp4"); await new Promise((resolve) => socket.bind(0, "127.0.0.1", resolve)); @@ -500,33 +508,32 @@ test("rudp session handshake acks reliable connect and accept packets", () => { test("rudp session computes ack masks and clears pending reliable packets", () => { const sender = new CultNetRudpSession({ connectionId: 7, initialSequence: 10, resendDelayMs: 100 }); const receiver = new CultNetRudpSession({ connectionId: 7, initialSequence: 200, resendDelayMs: 100 }); - sender.receive({ packetType: "accept", connectionId: 7, sequence: 1, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 7, sequence: 2, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const first = sender.send("schema", Buffer.from("first"), { reliable: true, ordered: true, nowMs: 0 }); const second = sender.send("schema", Buffer.from("second"), { reliable: true, ordered: true, nowMs: 0 }); const third = sender.send("schema", Buffer.from("third"), { reliable: true, ordered: true, nowMs: 0 }); - assert.deepEqual(sender.pendingReliableSequences, [10, 11, 12]); + assert.deepEqual(sender.pendingReliableSequences, [11, 12, 13]); receiver.receive(first); receiver.receive(third); const ackWithGap = receiver.createAck(); - assert.equal(ackWithGap.ack, 12); - assert.equal(ackWithGap.ackMask, 0b10 | (1 << 9)); + assert.equal(ackWithGap.ack, 13); + assert.equal(ackWithGap.ackMask, 0b110); sender.receive(ackWithGap); - assert.deepEqual(sender.pendingReliableSequences, [11]); + assert.deepEqual(sender.pendingReliableSequences, [12]); receiver.receive(second); const fullAck = receiver.createAck(); - assert.equal(fullAck.ack, 12); - assert.equal(fullAck.ackMask, 0b11 | (1 << 9)); + assert.equal(fullAck.ack, 13); + assert.equal(fullAck.ackMask, 0b111); sender.receive(fullAck); assert.deepEqual(sender.pendingReliableSequences, []); }); test("rudp session schedules reliable resends until acked", () => { const session = new CultNetRudpSession({ connectionId: 99, initialSequence: 1, resendDelayMs: 100 }); - session.receive({ packetType: "accept", connectionId: 99, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); + session.receive(new CultNetRudpSession({ connectionId: 99, initialSequence: 900 }).acceptConnect(session.createConnect(0), 0), 0); const sent = session.send("schema", Buffer.from("payload"), { reliable: true, ordered: true, nowMs: 10 }); assert.deepEqual(session.dueResends(90), []); @@ -597,7 +604,7 @@ test("rudp unreliable sequenced delivery is scoped to its channel", () => { test("rudp rejects unreliable ordered delivery", () => { const session = new CultNetRudpSession({ connectionId: 196 }); - session.receive({ packetType: "accept", connectionId: 196, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); + session.receive(new CultNetRudpSession({ connectionId: 196, initialSequence: 900 }).acceptConnect(session.createConnect(0), 0), 0); assert.throws( () => session.send("schema", Buffer.from("cannot order what will not retransmit"), { ordered: true }), /ordered delivery requires reliability/, @@ -606,7 +613,7 @@ test("rudp rejects unreliable ordered delivery", () => { test("rudp session bounds pending reliable packets before enqueue", () => { const session = new CultNetRudpSession({ connectionId: 102, initialSequence: 1, maxPendingReliablePackets: 2 }); - session.receive({ packetType: "accept", connectionId: 102, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); + session.receive(new CultNetRudpSession({ connectionId: 102, initialSequence: 900 }).acceptConnect(session.createConnect(0), 0), 0); session.send("schema", Buffer.from("first"), { reliable: true, ordered: true }); session.send("schema", Buffer.from("second"), { reliable: true, ordered: true }); @@ -614,10 +621,10 @@ test("rudp session bounds pending reliable packets before enqueue", () => { () => session.send("schema", Buffer.from("third"), { reliable: true, ordered: true }), /reliable send queue is full/, ); - assert.deepEqual(session.pendingReliableSequences, [1, 2]); + assert.deepEqual(session.pendingReliableSequences, [2, 3]); const fragmented = new CultNetRudpSession({ connectionId: 103, initialSequence: 1, maxPendingReliablePackets: 3 }); - fragmented.receive({ packetType: "accept", connectionId: 103, sequence: 50, ack: 0, ackMask: 0, channelId: "control" }); + fragmented.receive(new CultNetRudpSession({ connectionId: 103, initialSequence: 900 }).acceptConnect(fragmented.createConnect(0), 0), 0); assert.throws( () => fragmented.sendMany("schema", Buffer.from("fragment-me"), { reliable: true, ordered: true, maxFragmentBytes: 3 }), /reliable send queue is full/, @@ -628,8 +635,7 @@ test("rudp session bounds pending reliable packets before enqueue", () => { test("rudp session suppresses duplicates and delivers reliable ordered payloads in sequence", () => { const sender = new CultNetRudpSession({ connectionId: 123, initialSequence: 1 }); const receiver = new CultNetRudpSession({ connectionId: 123, initialSequence: 100 }); - sender.receive({ packetType: "accept", connectionId: 123, sequence: 90, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 123, sequence: 91, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0)); const first = sender.send("schema", Buffer.from("first"), { reliable: true, ordered: true }); const second = sender.send("schema", Buffer.from("second"), { reliable: true, ordered: true }); @@ -647,8 +653,7 @@ test("rudp session suppresses duplicates and delivers reliable ordered payloads test("rudp session skips received control packets while ordering schema payloads", () => { const sender = new CultNetRudpSession({ connectionId: 124, initialSequence: 1 }); const receiver = new CultNetRudpSession({ connectionId: 124, initialSequence: 100 }); - sender.receive({ packetType: "accept", connectionId: 124, sequence: 90, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 124, sequence: 91, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const first = sender.send("schema", Buffer.from("first"), { reliable: true, ordered: true }); const control = sender.createAck(); @@ -665,8 +670,7 @@ test("rudp session skips received control packets while ordering schema payloads test("rudp session fragments and reassembles reliable ordered payloads", () => { const sender = new CultNetRudpSession({ connectionId: 456, initialSequence: 1 }); const receiver = new CultNetRudpSession({ connectionId: 456, initialSequence: 100 }); - sender.receive({ packetType: "accept", connectionId: 456, sequence: 90, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 456, sequence: 91, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const packets = sender.sendMany("schema", Buffer.from("fragment-me-please"), { reliable: true, @@ -1375,8 +1379,7 @@ test("rudp session advances large fragment sets through a bounded reliable windo const connectionId = 457; const sender = new CultNetRudpSession({ connectionId, initialSequence: 1, resendDelayMs: 25 }); const receiver = new CultNetRudpSession({ connectionId, initialSequence: 100, resendDelayMs: 25 }); - sender.receive({ packetType: "accept", connectionId, sequence: 0, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId, sequence: 0, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const fragmentCount = CULTNET_RUDP_RELIABLE_SEND_WINDOW_PACKETS + 17; const payload = Uint8Array.from( @@ -1526,14 +1529,13 @@ test("RUDP operation service correlates typed payload envelopes", async () => { test("rudp sequence-neutral acknowledgements interoperate with ordered receivers", () => { const sender = new CultNetRudpSession({ connectionId: 125, initialSequence: 1 }); const receiver = new CultNetRudpSession({ connectionId: 125, initialSequence: 100 }); - sender.receive({ packetType: "accept", connectionId: 125, sequence: 90, ack: 0, ackMask: 0, channelId: "control" }); - receiver.receive({ packetType: "accept", connectionId: 125, sequence: 91, ack: 0, ackMask: 0, channelId: "control" }); + sender.receive(receiver.acceptConnect(sender.createConnect(0), 0), 0); const ack = sender.createAck(); const request = sender.send("schema", Buffer.from("snapshot"), { reliable: true, ordered: true }); assert.equal(ack.sequence, 0); - assert.equal(request.sequence, 1); + assert.equal(request.sequence, 2); assert.deepEqual(receiver.receive(ack).delivered, []); assert.deepEqual(receiver.receive(request).delivered.map(frame => Buffer.from(frame.payload).toString("utf8")), ["snapshot"]); }); @@ -1832,7 +1834,7 @@ test("rudp receiver evicts stranded fragment sets instead of refusing new ones", // Heimdall fragmented envelopes (max_fragment_bytes = 2048), so on the live // auth path this map grew for every lost fragment and was never swept. const receiver = new CultNetRudpSession({ connectionId: 9, initialSequence: 1, resendDelayMs: 50 }); - receiver.receive({ packetType: "accept", connectionId: 9, sequence: 1, ack: 0, ackMask: 0, channelId: "control" }); + receiver.acceptConnect(new CultNetRudpSession({ connectionId: 9, initialSequence: 1 }).createConnect(0), 0); // 200 fragment sets that each lose their second fragment. for (let id = 1; id <= 200; id += 1) { @@ -1976,7 +1978,7 @@ test("rudp socket transport ends the session that refuses a reliable frame and n // neither in the goodbye nor after it retransmits. const goodbye = received.find((p) => p.packetType === "disconnect")!; assert.equal(peer.receive(goodbye, 1).disconnected, true); - assert.ok(peer.pendingReliableSequences.includes(poison.sequence), "the refused frame was acknowledged"); + assert.equal(acknowledges(goodbye, poison.sequence), false, "the goodbye acknowledged the refused frame"); const before = received.length; toServer(poison); await waitFor(() => server.stats.packetsDropped === 2, "the retransmit dropped"); @@ -2353,7 +2355,7 @@ test("a repeated Connect acknowledges what it carries", () => { const server = new CultNetRudpSession({ connectionId }); const connect = client.createConnect(0); const accept = server.acceptConnect(connect, 0); - const reply = server.answerRepeatedConnect({ ...connect, ack: accept.sequence }, 1); + const reply = server.acceptConnect({ ...connect, ack: accept.sequence }, 1); assert.equal(reply.packetType, "ack"); assert.equal(server.outstandingReliablePacketCount, 0); }); @@ -2490,3 +2492,190 @@ test("operation service sweeps a session that ran a handler once it idles out", await server.close(); } }); + +test("a flush started after a reconnect waits on the new session", async () => { + const peerSocket = await bindUdpSocket(); + const clientSocket = await bindUdpSocket(); + const connectionId = 0x1020305b; + const client = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-client", + socket: clientSocket, + mode: "client", + remoteHost: "127.0.0.1", + remotePort: udpPort(peerSocket), + connectionId, + resendDelayMs: 1000, + resendPollMs: 1000, + }); + const received: CultNetRudpPacket[] = []; + let clientPort = 0; + peerSocket.on("message", (wire, remote) => { + clientPort = remote.port; + received.push(decodeRudpPacket(wire)); + }); + try { + client.connect(); + await waitFor(() => received.some((p) => p.packetType === "connect"), "the Connect"); + const first = new CultNetRudpSession({ connectionId }); + peerSocket.send(encodeRudpPacket(first.acceptConnect(received[0]!, 0)), clientPort, "127.0.0.1"); + await waitFor(() => client.connected, "the client connected"); + client.send("schema", Buffer.from("owed to the old session")); + client.checkTimeout(50, Date.now() + 10_000); + assert.equal(client.connected, false); + + client.connect(); + await waitFor(() => received.filter((p) => p.packetType === "connect").length === 2, "the reconnect"); + const second = new CultNetRudpSession({ connectionId }); + const reconnect = received.filter((p) => p.packetType === "connect")[1]!; + peerSocket.send(encodeRudpPacket(second.acceptConnect(reconnect, 0)), clientPort, "127.0.0.1"); + await waitFor(() => client.connected, "the client reconnected"); + await client.flush(500); + } finally { + peerSocket.close(); + client.close(); + } +}); + +test("a flush started after the server accepts a new Connect waits on the new session", async () => { + const serverSocket = await bindUdpSocket(); + const peerSocket = await bindUdpSocket(); + const connectionId = 0x1020305c; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendDelayMs: 1000, + resendPollMs: 1000, + }); + const received: CultNetRudpPacket[] = []; + peerSocket.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const toServer = (packet: CultNetRudpPacket) => + peerSocket.send(encodeRudpPacket(packet), udpPort(serverSocket), "127.0.0.1"); + try { + const first = new CultNetRudpSession({ connectionId }); + toServer(first.createConnect(0)); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the first Accept"); + toServer(first.createDisconnect(Buffer.from("bye"))); + await waitFor(() => !server.connected, "the first session ended"); + + const next = new CultNetRudpSession({ connectionId }); + toServer(next.createConnect(0)); + await waitFor(() => received.filter((p) => p.packetType === "accept").length === 2, "the second Accept"); + const accept = received.filter((p) => p.packetType === "accept")[1]!; + next.receive(accept, 0); + toServer(next.createAckForReceived(accept.sequence)); + await waitFor(() => server.outstandingReliablePacketCount === 0, "the second Accept acknowledged"); + + await server.flush(500); + } finally { + peerSocket.close(); + server.close(); + } +}); + +test("server-mode transport admits a restarted client and keeps its session when a Connect is retransmitted", async () => { + const serverSocket = await bindUdpSocket(); + const peerSocket = await bindUdpSocket(); + const connectionId = 0x10203059; + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "rudp-server", + socket: serverSocket, + mode: "server", + connectionId, + resendPollMs: 1000, + }); + let received: CultNetRudpPacket[] = []; + peerSocket.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const frames: string[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(Buffer.from(frame.payload).toString("utf8"))); + const toServer = (packet: CultNetRudpPacket) => + peerSocket.send(encodeRudpPacket(packet), udpPort(serverSocket), "127.0.0.1"); + const reliable = { reliable: true, ordered: true }; + try { + const first = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const connect = first.createConnect(0, Buffer.from("same")); + toServer(connect); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the first Accept"); + first.receive(received.find((p) => p.packetType === "accept")!, 0); + const sent = [0, 1, 2].map((index) => first.send("schema", Buffer.from(`c${index}`), reliable)); + sent.forEach(toServer); + await waitFor(() => frames.length === 3, "the first client's frames"); + + // A retransmitted Connect is a repeat: what was delivered stays delivered once. + toServer(connect); + toServer(sent[0]!); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.deepEqual(frames, ["c0", "c1", "c2"]); + + // The same client restarted on the same address, connection id and payload. + received = []; + const second = new CultNetRudpSession({ connectionId, initialSequence: 7 }); + toServer(second.createConnect(0, Buffer.from("same"))); + await waitFor(() => received.some((p) => p.packetType === "accept"), "the restarted client's Accept"); + second.receive(received.find((p) => p.packetType === "accept")!, 0); + toServer(second.send("schema", Buffer.from("after restart"), reliable)); + await waitFor(() => frames.includes("after restart"), "the restarted client's frame"); + } finally { + peerSocket.close(); + server.close(); + } +}); + +test("RUDP operation service admits a restarted client on the same address", async () => { + const server = await startCultNetOperationServer({ + runtimeId: "sai-sidecar", + handler: request => ({ + schemaVersion: "cultnet.operation_response.v0", + messageId: request.messageId, + serviceId: request.serviceId, + operation: request.operation, + status: "ok", + payloadSchema: "gamecult.eve.plugin_abi.response.v1", + payloadEncoding: "messagepack-base64", + payload: request.payload, + diagnostics: [], + sourceRuntimeId: "sai-sidecar", + }), + }); + const connectionId = 0x43554c54; + const port = Number(new URL(server.endpoint).port); + const socket = await bindUdpSocket(); + let session = new CultNetRudpSession({ connectionId, initialSequence: 50_000 }); + const answered: string[] = []; + socket.on("message", (wire) => { + const packet = decodeRudpPacket(wire); + const result = session.receive(packet, Date.now()); + if (packet.packetType === "accept" || packet.reliable) { + socket.send(encodeRudpPacket(session.createAckForReceived(packet.sequence)), port, "127.0.0.1"); + } + for (const frame of result.delivered) { + answered.push((decodeMsgpack(frame.payload) as { messageId: string }).messageId); + } + }); + const toServer = (packet: CultNetRudpPacket) => socket.send(encodeRudpPacket(packet), port, "127.0.0.1"); + const request = (messageId: string) => Buffer.from(encode(encodeCultNetMessageForWire({ + schemaVersion: "cultnet.operation_request.v0", + messageId, + serviceId: "sai.vn", + operation: "describe", + payloadSchema: "gamecult.eve.plugin_abi.request.v1", + payloadEncoding: "messagepack-base64", + payload: "gaZzY2hlbWE=", + } satisfies CultNetOperationRequestMessage))); + try { + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the first Accept"); + toServer(session.send("schema", request("before-restart"), { reliable: true, ordered: true })); + await waitFor(() => answered.includes("before-restart"), "the first answer"); + + session = new CultNetRudpSession({ connectionId, initialSequence: 7 }); + toServer(session.createConnect(Date.now())); + await waitFor(() => session.connected, "the restarted client's Accept"); + toServer(session.send("schema", request("after-restart"), { reliable: true, ordered: true })); + await waitFor(() => answered.includes("after-restart"), "the restarted client's answer"); + } finally { + socket.close(); + await server.close(); + } +}); diff --git a/packages/cultnet-ts/test/rudp-connect-generation.test.ts b/packages/cultnet-ts/test/rudp-connect-generation.test.ts new file mode 100644 index 00000000..cea61a6d --- /dev/null +++ b/packages/cultnet-ts/test/rudp-connect-generation.test.ts @@ -0,0 +1,535 @@ +// A Connect's sequence says whether it repeats the one a session accepted, and +// the handshake alone seeds the watermark that orders delivery. +import assert from "node:assert/strict"; +import test from "node:test"; +import { createSocket, type Socket } from "node:dgram"; +import { once } from "node:events"; +import { + CultNetRudpSession, + CultNetRudpSocketTransportConnection, + decodeRudpPacket, + encodeRudpPacket, + type CultNetRudpPacket, +} from "../src"; + +const connectionId = 0x10203080; +const enc = (text: string) => new TextEncoder().encode(text); +const dec = (bytes: Uint8Array) => new TextDecoder().decode(bytes); +const session = (initialSequence: number) => new CultNetRudpSession({ connectionId, initialSequence }); +const send = (from: CultNetRudpSession, text: string) => from.send("schema", enc(text), { reliable: true, ordered: true }); +const names = (result: { delivered: { payload: Uint8Array }[] }) => result.delivered.map((frame) => dec(frame.payload)); + +function handshake(client: CultNetRudpSession, server: CultNetRudpSession): { connect: CultNetRudpPacket; accept: CultNetRudpPacket } { + const connect = client.createConnect(0); + const accept = server.acceptConnect(connect, 0); + client.receive(accept, 0); + assert.ok(client.connected && server.connected); + return { connect, accept }; +} + +test("a reconnect on the same session is not stranded by a write the ended session owed", () => { + const client = session(1); + const server = session(500); + handshake(client, server); + send(server, "lost"); + server.createDisconnect(enc("gone")); + + client.receive(server.acceptConnect(client.createConnect(1), 1), 1); + assert.deepEqual(names(client.receive(send(server, "after"), 2)), ["after"]); +}); + +test("a server that accepts a new client forgets the old client's sequences", () => { + const oldClient = session(50); + const server = session(500); + handshake(oldClient, server); + for (const name of ["c1", "c2", "c3"]) { + assert.deepEqual(names(server.receive(send(oldClient, name), 1)), [name]); + } + server.createDisconnect(enc("restart")); + + const newClient = session(2); + newClient.receive(server.acceptConnect(newClient.createConnect(2), 2), 2); + assert.deepEqual(names(server.receive(send(newClient, "fresh"), 3)), ["fresh"]); +}); + +test("a retransmitted Connect after data has flowed does not reset the session", () => { + const client = session(1); + const server = session(500); + const { connect } = handshake(client, server); + const frame = send(client, "once"); + assert.deepEqual(names(server.receive(frame, 1)), ["once"]); + + assert.equal(server.connectRepeats(connect), true); + assert.equal(server.acceptConnect(connect, 2).packetType, "ack", "the Accept was already acknowledged"); + assert.equal(server.outstandingReliablePacketCount, 0, "a repeat queued something"); + assert.deepEqual(server.receive(frame, 3).delivered, [], "a retransmitted Connect made the server forget what it had delivered"); +}); + +test("a Connect with another sequence is not a repeat whatever it carries", () => { + const server = session(500); + const connect = session(1).createConnect(0, enc("same payload")); + server.acceptConnect(connect, 0); + assert.equal(server.connectRepeats(connect), true); + + assert.equal(server.connectRepeats(session(9).createConnect(0, enc("same payload"))), false); + assert.equal(server.connectRepeats({ ...connect, packetType: "ping" }), false); + server.createDisconnect(); + assert.equal(server.connectRepeats(connect), false, "an ended session repeats nothing"); +}); + +test("an Accept that does not name the pending Connect is ignored", () => { + const client = session(10); + const server = session(500); + const connect = client.createConnect(0); + const accept = server.acceptConnect(connect, 0); + + client.receive({ ...accept, ack: connect.sequence + 50, ackMask: 0 }, 1); + assert.equal(client.connected, false, "a stale Accept connected the session"); + assert.deepEqual(client.pendingReliableSequences, [connect.sequence]); + + client.receive({ ...accept, ack: connect.sequence + 3, ackMask: 1 << 2 }, 2); + assert.equal(client.connected, true, "the mask names the Connect"); +}); + +test("an Accept after a local disconnect does not reconnect the session", () => { + const client = session(1); + const server = session(500); + const { accept } = handshake(client, server); + client.createDisconnect(enc("bye")); + assert.equal(client.connected, false); + + client.receive(accept, 5); + assert.equal(client.connected, false, "a late Accept revived an ended session"); +}); + +test("a reconnect forgets what the old server sent", () => { + const client = session(1); + const oldServer = session(500); + handshake(client, oldServer); + for (const name of ["d1", "d2", "d3"]) { + assert.deepEqual(names(client.receive(send(oldServer, name), 1)), [name]); + } + + const newServer = session(501); + client.receive(newServer.acceptConnect(client.createConnect(2), 2), 2); + assert.deepEqual(names(client.receive(send(newServer, "fresh"), 3)), ["fresh"]); +}); + +test("an Accept for an abandoned Connect does not connect the session", () => { + const client = session(1); + const server = session(500); + const accept = server.acceptConnect(client.createConnect(0), 0); + client.createDisconnect(enc("never mind")); + + client.receive(accept, 1); + assert.equal(client.connected, false, "an Accept revived an abandoned Connect"); +}); + +test("a full queue owed to a vanished peer does not refuse a new generation", () => { + const bounded = (initialSequence: number) => new CultNetRudpSession({ connectionId, initialSequence, maxPendingReliablePackets: 1 }); + const server = bounded(500); + server.acceptConnect(session(1).createConnect(0), 0); + assert.equal(server.outstandingReliablePacketCount, 1, "the Accept is owed and never acknowledged"); + const next = session(9).createConnect(1); + server.acceptConnect(next, 1); + assert.equal(server.connectRepeats(next), true); + + const client = bounded(1); + client.createConnect(0); + client.createConnect(1); + assert.equal(client.outstandingReliablePacketCount, 1, "only the new Connect is owed"); +}); + +test("a duplicate Accept does not reseed the watermark", () => { + const client = session(1); + const server = session(500); + const { accept } = handshake(client, server); + const s1 = send(server, "s1"); + const s2 = send(server, "s2"); + client.receive({ ...accept, sequence: s2.sequence }, 1); + assert.deepEqual(client.receive(s2, 2).delivered, [], "s1 is still missing"); + assert.deepEqual(names(client.receive(s1, 3)), ["s1", "s2"]); +}); + +test("reliable data before the Accept is neither delivered nor acknowledged", () => { + const client = session(1); + const server = session(500); + const accept = server.acceptConnect(client.createConnect(0), 0); + const a = send(server, "A"); + const b = send(server, "B"); + + for (const early of [b, a]) { + assert.deepEqual(client.receive(early, 1).delivered, []); + const ack = client.createAckForReceived(early.sequence); + assert.deepEqual([ack.ack, ack.ackMask], [0, 0], "an unhandled packet was acknowledged"); + } + client.receive(accept, 2); + assert.ok(client.connected); + assert.equal(server.outstandingReliablePacketCount, 3, "nothing was acknowledged"); + + assert.deepEqual(names(client.receive(a, 3)), ["A"]); + assert.deepEqual(names(client.receive(b, 3)), ["B"]); +}); + +test("ordered frames after the Accept are delivered in order whatever arrives first", () => { + const client = session(1); + const server = session(500); + handshake(client, server); + const a = send(server, "A"); + const b = send(server, "B"); + assert.deepEqual(client.receive(b, 1).delivered, []); + assert.deepEqual(names(client.receive(a, 2)), ["A", "B"]); +}); + +test("an unordered reliable frame is delivered on receipt, never held behind a gap", () => { + const client = session(1); + const server = session(500); + handshake(client, server); + send(server, "lost"); + const unordered = server.send("media", enc("now"), { reliable: true, ordered: false }); + const ordered = send(server, "later"); + assert.deepEqual(names(client.receive(unordered, 1)), ["now"]); + assert.deepEqual(client.receive(ordered, 2).delivered, [], "the ordered frame waits for the gap"); +}); + +test("a peer Disconnect does not forget what the peer sent", () => { + const client = session(1); + const server = session(500); + handshake(client, server); + const s1 = send(client, "s1"); + const s2 = send(client, "s2"); + assert.deepEqual(server.receive(s2, 1).delivered, [], "s1 is missing, so s2 is held"); + + server.receive(client.createDisconnect(enc("bye")), 2); + assert.deepEqual(names(server.receive(s1, 3)), ["s1", "s2"], "the Disconnect forgot the held frame"); + assert.deepEqual(server.receive(s2, 4).delivered, [], "the Disconnect forgot what was received"); +}); + +test("sessions from one options object each draw their own initial sequence from one to two to the thirty first", () => { + const options = { connectionId }; + const firsts = new Set(Array.from({ length: 64 }, () => new CultNetRudpSession(options).createConnect(0).sequence)); + assert.ok(firsts.size > 32, `a default session does not draw its own sequence: ${[...firsts]}`); + assert.ok([...firsts].every((value) => value >= 1 && value < 2 ** 31), `left [1, 2^31): ${[...firsts]}`); +}); + +// Ack Cut 1d: a frame from an earlier generation is a duplicate, a delayed +// Connect from an earlier attempt is stale, a Connect from a new endpoint is a +// new client. + +const namesSequence = (ack: CultNetRudpPacket, sequence: number) => + ack.ack === sequence + || Array.from({ length: 32 }, (_, bit) => bit).some( + (bit) => (ack.ackMask & (1 << bit)) !== 0 && ack.ack - bit - 1 === sequence, + ); + +test("a frame from the client's earlier generation is acknowledged, not delivered again", () => { + const client = session(10); + const server = session(500); + handshake(client, server); + const old = send(client, "old"); + assert.deepEqual(names(server.receive(old, 1)), ["old"]); + + const connect = client.createConnect(2); + client.receive(server.acceptConnect(connect, 2), 2); + assert.ok(old.sequence < connect.sequence); + assert.deepEqual(server.receive(old, 3).delivered, [], "an earlier generation's frame was delivered again"); + assert.ok(namesSequence(server.createAckForReceived(old.sequence), old.sequence), "the duplicate was not acknowledged"); + assert.deepEqual(names(server.receive(send(client, "fresh"), 4)), ["fresh"]); +}); + +test("a frame from the server's earlier generation is acknowledged, not delivered again", () => { + const client = session(10); + const server = session(500); + handshake(client, server); + const old = send(server, "old"); + assert.deepEqual(names(client.receive(old, 1)), ["old"]); + + client.receive(server.acceptConnect(client.createConnect(2), 2), 2); + assert.deepEqual(client.receive(old, 3).delivered, [], "an earlier generation's frame was delivered again"); + assert.ok(namesSequence(client.createAckForReceived(old.sequence), old.sequence)); + assert.deepEqual(names(client.receive(send(server, "fresh"), 4)), ["fresh"]); +}); + +test("a duplicate below the receive window is acknowledged by name", () => { + const client = session(10); + const server = session(500); + handshake(client, server); + const first = send(client, "first"); + assert.deepEqual(names(server.receive(first, 1)), ["first"]); + for (let index = 0; index < 4_200; index += 1) { + server.receive(send(client, "x"), 1); + client.receive(server.createAck(), 1); + } + assert.deepEqual(server.receive(first, 2).delivered, []); + assert.equal(server.createAckForReceived(first.sequence).ack, first.sequence, "a duplicate below the window was not acknowledged by name"); +}); + +test("a delayed Connect from an earlier attempt does not strand the client", () => { + const client = session(10); + const server = session(500); + const earlier = client.createConnect(0); + const retried = client.createConnect(300); + const accept = server.acceptConnect(retried, 301); + client.receive(accept, 302); + server.receive(client.createAckForReceived(accept.sequence), 302); + assert.ok(client.connected); + const a = send(client, "a"); + assert.deepEqual(names(server.receive(a, 303)), ["a"]); + client.receive(server.createAckForReceived(a.sequence), 303); + + assert.equal(server.connectRepeats(earlier), true, "the delayed Connect starts nothing"); + const reply = server.acceptConnect(earlier, 304); + assert.equal(reply.packetType, "ack"); + client.receive(reply, 305); + assert.ok(client.connected); + + const b = send(client, "b"); + assert.deepEqual(names(server.receive(b, 306)), ["b"], "the delayed Connect reset the server"); + client.receive(server.createAckForReceived(b.sequence), 307); + assert.ok(!client.pendingReliableSequences.includes(b.sequence)); +}); + +test("a repeated Connect refreshes liveness and a stale one does not", () => { + const client = session(10); + const server = session(500); + const earlier = client.createConnect(0); + const current = client.createConnect(1); + server.acceptConnect(current, 0); + + server.acceptConnect(current, 900); + assert.equal(server.checkTimeout(1_000, 500), false, "a repeated Connect did not refresh liveness"); + + server.acceptConnect(earlier, 1_400); + assert.equal(server.checkTimeout(1_600, 500), true, "a stale Connect refreshed liveness"); +}); + +test("stale Connects are recognised by serial arithmetic", () => { + const server = session(500); + server.acceptConnect(session(3).createConnect(0), 0); + const connect = (sequence: number) => session(sequence).createConnect(0); + assert.equal(server.connectRepeats(connect(2 ** 32 - 2)), true, "just before the wrap"); + assert.equal(server.connectRepeats(connect(4)), false, "just after"); + assert.equal(server.connectRepeats(connect(3 + 4_096)), false, "far after"); + assert.equal(server.connectRepeats(connect(2 ** 32 - 4_092)), true, "the window's edge"); + assert.equal(server.connectRepeats(connect(2 ** 32 - 4_093)), false, "past the window"); +}); + +test("a restarted client whose first sequence is stale connects after redrawing", () => { + const server = session(500); + const old = session(1_000); + const oldAccept = server.acceptConnect(old.createConnect(0), 0); + old.receive(oldAccept, 0); + + const restarted = session(900); + const first = restarted.createConnect(0, enc("join")); + // The old client has not acknowledged its Accept yet, so that is the reply: + // it names the old Connect, not this one. + const early = server.acceptConnect(first, 1); + assert.equal(early.packetType, "accept"); + restarted.receive(early, 1); + assert.equal(restarted.connected, false, "an Accept for another Connect connected the client"); + server.receive(old.createAckForReceived(oldAccept.sequence), 1); + const reply = server.acceptConnect(first, 2); + assert.equal(reply.packetType, "ack"); + restarted.receive(reply, 2); + assert.equal(restarted.connected, false); + + const retransmitted = restarted.dueResends(1_000); + assert.equal(retransmitted.length, 1); + assert.equal(retransmitted[0]!.sequence, first.sequence, "the attempt is still young"); + + const fresh = restarted.dueResends(3_500); + assert.equal(fresh.length, 1); + assert.equal(fresh[0]!.packetType, "connect"); + assert.notEqual(fresh[0]!.sequence, first.sequence, "the same Connect was retransmitted for ever"); + assert.equal(dec(fresh[0]!.payload ?? new Uint8Array()), "join", "the fresh attempt lost the Connect payload"); + + const accept = server.acceptConnect(fresh[0]!, 3_500); + assert.equal(accept.packetType, "accept"); + restarted.receive(accept, 3_500); + assert.ok(restarted.connected); + assert.deepEqual(names(server.receive(send(restarted, "hello"), 3_501)), ["hello"]); +}); + +test("an answered Connect is never replaced by the attempt timeout", () => { + const client = session(10); + const server = session(500); + const { connect } = handshake(client, server); + const resent = client.dueResends(60_000); + assert.ok(resent.every((packet) => packet.packetType !== "connect" || packet.sequence === connect.sequence), "a connected client started a new attempt"); + assert.ok(client.connected); +}); + +async function bind(): Promise { + const socket = createSocket("udp4"); + socket.bind(0, "127.0.0.1"); + await once(socket, "listening"); + return socket; +} +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test("server mode admits a pinned client that restarts on a new port", async () => { + const serverSocket = await bind(); + const a = await bind(); + const b = await bind(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "srv", socket: serverSocket, mode: "server", connectionId, resendPollMs: 20, resendDelayMs: 20, + }); + const frames: string[] = []; + server.on("frame", (frame: { payload: Uint8Array }) => frames.push(dec(frame.payload))); + const port = (serverSocket.address() as { port: number }).port; + const atA: CultNetRudpPacket[] = []; + const atB: CultNetRudpPacket[] = []; + a.on("message", (wire) => atA.push(decodeRudpPacket(wire))); + b.on("message", (wire) => atB.push(decodeRudpPacket(wire))); + try { + const first = session(1); + a.send(encodeRudpPacket(first.createConnect(0, enc("join"))), port, "127.0.0.1"); + await sleep(100); + const accept = atA.find((packet) => packet.packetType === "accept"); + assert.ok(accept); + first.receive(accept, 0); + a.send(encodeRudpPacket(first.createAckForReceived(accept.sequence)), port, "127.0.0.1"); + a.send(encodeRudpPacket(send(first, "hello")), port, "127.0.0.1"); + await sleep(100); + assert.deepEqual(frames, ["hello"]); + + const second = session(1); // the process restarted on a new port, pinned + b.send(encodeRudpPacket(second.createConnect(0, enc("join"))), port, "127.0.0.1"); + await sleep(100); + const acceptB = atB.find((packet) => packet.packetType === "accept"); + assert.ok(acceptB, `the restarted client on a new port was not admitted: ${JSON.stringify(atB.map((packet) => packet.packetType))}`); + second.receive(acceptB, 1); + assert.ok(second.connected); + b.send(encodeRudpPacket(send(second, "after")), port, "127.0.0.1"); + await sleep(100); + assert.deepEqual(frames, ["hello", "after"]); + } finally { + server.close(); + a.close(); + b.close(); + } +}); + +test("an accepted peer that goes silent times out", () => { + const server = session(500); + server.acceptConnect(session(10).createConnect(0), 0); + assert.equal(server.checkTimeout(100_000, 1_000), true); +}); + +// Ack Cut 1d, batch 2: the connect-attempt timeout. + +test("an Ack naming the pending Connect does not retire it", () => { + const server = session(500); + const old = session(1); + const oldAccept = server.acceptConnect(old.createConnect(0), 0); + old.receive(oldAccept, 0); + server.receive(old.createAckForReceived(oldAccept.sequence), 0); + + const restarted = session(1); + const first = restarted.createConnect(0, enc("join")); + const reply = server.acceptConnect(first, 1); + assert.equal(reply.packetType, "ack"); + assert.ok(namesSequence(reply, first.sequence), "the Ack must name the pending Connect for this test to bite"); + restarted.receive(reply, 1); + assert.equal(restarted.connected, false); + assert.deepEqual(restarted.pendingReliableSequences, [first.sequence], "an Ack retired the Connect"); + + const fresh = restarted.dueResends(3_000); + assert.equal(fresh.length, 1, "the client waits for ever with nothing to resend"); + assert.equal(dec(fresh[0]!.payload ?? new Uint8Array()), "join"); + const accept = server.acceptConnect(fresh[0]!, 3_000); + assert.equal(accept.packetType, "accept"); + restarted.receive(accept, 3_000); + assert.ok(restarted.connected); +}); + +test("a late copy of an abandoned Connect is stale once its replacement is accepted", () => { + const server = session(500); + const client = session(10); + const abandoned = client.createConnect(0); + const fresh = client.dueResends(3_000); + assert.equal(fresh.length, 1); + assert.equal(fresh[0]!.sequence, abandoned.sequence + 4_095); + const accept = server.acceptConnect(fresh[0]!, 3_001); + client.receive(accept, 3_002); + server.receive(client.createAckForReceived(accept.sequence), 3_002); + assert.ok(client.connected); + const a = send(client, "a"); + assert.deepEqual(names(server.receive(a, 3_003)), ["a"]); + client.receive(server.createAckForReceived(a.sequence), 3_003); + + assert.equal(server.connectRepeats(abandoned), true, "the late copy would restart the server"); + const reply = server.acceptConnect(abandoned, 3_004); + assert.equal(reply.packetType, "ack"); + client.receive(reply, 3_005); + assert.ok(client.connected); + assert.deepEqual(names(server.receive(send(client, "b"), 3_006)), ["b"]); +}); + +test("a server sitting at the jump is left by the next attempt", () => { + const server = session(500); + const old = session(5_095); + const oldAccept = server.acceptConnect(old.createConnect(0), 0); + old.receive(oldAccept, 0); + server.receive(old.createAckForReceived(oldAccept.sequence), 0); + + const restarted = session(1_000); + const first = restarted.createConnect(0, enc("join")); + restarted.receive(server.acceptConnect(first, 1), 1); + assert.equal(restarted.connected, false); + + const second = restarted.dueResends(3_000)[0]!; + assert.equal(second.sequence, 5_095, "the jump lands on the server's generation"); + const reply = server.acceptConnect(second, 3_001); + assert.equal(reply.packetType, "ack"); + restarted.receive(reply, 3_001); + assert.equal(restarted.connected, false); + + const third = restarted.dueResends(6_000)[0]!; + assert.equal(third.sequence, 5_095 + 4_095); + const accept = server.acceptConnect(third, 6_001); + assert.equal(accept.packetType, "accept"); + restarted.receive(accept, 6_001); + assert.ok(restarted.connected); +}); + +test("server mode admits a pinned client that restarts on the same address", async () => { + const serverSocket = await bind(); + const peer = await bind(); + const server = new CultNetRudpSocketTransportConnection({ + runtimeId: "srv", socket: serverSocket, mode: "server", connectionId, resendPollMs: 20, resendDelayMs: 20, + }); + const port = (serverSocket.address() as { port: number }).port; + const received: CultNetRudpPacket[] = []; + peer.on("message", (wire) => received.push(decodeRudpPacket(wire))); + const take = async (type: string) => { + for (let index = 0; index < 50; index += 1) { + const found = received.findIndex((packet) => packet.packetType === type); + if (found >= 0) return received.splice(found, 1)[0]!; + await sleep(10); + } + throw new Error(`no ${type} arrived`); + }; + try { + const first = session(1); + peer.send(encodeRudpPacket(first.createConnect(0, enc("join"))), port, "127.0.0.1"); + const accept = await take("accept"); + first.receive(accept, 0); + peer.send(encodeRudpPacket(first.createAckForReceived(accept.sequence)), port, "127.0.0.1"); + await sleep(50); + received.length = 0; + + const restarted = session(1); + peer.send(encodeRudpPacket(restarted.createConnect(0, enc("join"))), port, "127.0.0.1"); + restarted.receive(await take("ack"), 1); + assert.equal(restarted.connected, false); + + peer.send(encodeRudpPacket(restarted.dueResends(3_000)[0]!), port, "127.0.0.1"); + restarted.receive(await take("accept"), 3_001); + assert.ok(restarted.connected, "the restarted pinned client was never admitted"); + } finally { + server.close(); + peer.close(); + } +}); diff --git a/packages/cultnet-ts/test/rudp-ordered-delivery.test.ts b/packages/cultnet-ts/test/rudp-ordered-delivery.test.ts new file mode 100644 index 00000000..519fc8aa --- /dev/null +++ b/packages/cultnet-ts/test/rudp-ordered-delivery.test.ts @@ -0,0 +1,110 @@ +// Ordered delivery has one owner: the contiguous received-through watermark. A +// frame is held for exactly one reason, a reliable sequence below it has not +// arrived, and it is delivered in the call that fills the last such gap, +// whichever channel the gap belonged to. +import assert from "node:assert/strict"; +import test from "node:test"; +import { CultNetRudpSession } from "../src"; + +const enc = (text: string) => new TextEncoder().encode(text); +const dec = (bytes: Uint8Array) => new TextDecoder().decode(bytes); + +function handshake(): { client: CultNetRudpSession; server: CultNetRudpSession } { + const client = new CultNetRudpSession({ connectionId: 410, initialSequence: 1 }); + const server = new CultNetRudpSession({ connectionId: 410, initialSequence: 500 }); + const accept = server.acceptConnect(client.createConnect(0), 0); + client.receive(accept, 0); + return { client, server }; +} + +const names = (result: { delivered: { payload: Uint8Array }[] }) => result.delivered.map((frame) => dec(frame.payload)); +const send = (session: CultNetRudpSession, channel: string, payload: string, ordered: boolean) => + session.send(channel, enc(payload), { reliable: true, ordered }); + +test("an ordered frame waits for a gap filled by another channel", () => { + const { client: sender, server: receiver } = handshake(); + const o1 = send(sender, "schema", "o1", true); + const u = send(sender, "rel", "u", false); + const o2 = send(sender, "schema", "o2", true); + const o3 = send(sender, "schema", "o3", true); + const o4 = send(sender, "schema", "o4", true); + + assert.deepEqual(names(receiver.receive(o1, 1)), ["o1"]); + assert.deepEqual(names(receiver.receive(o2, 2)), [], "u is missing"); + assert.deepEqual(names(receiver.receive(o3, 3)), [], "u is missing"); + assert.deepEqual(names(receiver.receive(o4, 4)), [], "u is missing"); + assert.deepEqual(names(receiver.receive(u, 5)), ["u", "o2", "o3", "o4"]); +}); + +test("two ordered channels release each other in sequence order", () => { + const { client: sender, server: receiver } = handshake(); + const a1 = send(sender, "schema", "A1", true); + const b1 = send(sender, "other", "B1", true); + const a2 = send(sender, "schema", "A2", true); + const a3 = send(sender, "schema", "A3", true); + + const delivered = [ + ...names(receiver.receive(a1, 1)), + ...names(receiver.receive(a2, 2)), + ...names(receiver.receive(a3, 3)), + ]; + assert.deepEqual(delivered, ["A1"], "B1 is missing, so A2 and A3 wait"); + assert.deepEqual(names(receiver.receive(b1, 4)), ["B1", "A2", "A3"]); +}); + +test("a channel first used after other traffic loses nothing", () => { + const { client: sender, server: receiver } = handshake(); + const c1 = send(sender, "late", "C1", true); + const c2 = send(sender, "late", "C2", true); + const x = send(sender, "schema", "X", true); + + assert.deepEqual(names(receiver.receive(x, 1)), [], "C1 is missing"); + assert.deepEqual(names(receiver.receive(c2, 2)), [], "C1 is missing"); + assert.deepEqual(names(receiver.receive(c1, 3)), ["C1", "C2", "X"]); +}); + +test("Accept seeds the watermark of the connecting side", () => { + const { client, server } = handshake(); + const s1 = send(server, "schema", "s1", true); + const s2 = send(server, "schema", "s2", true); + + assert.deepEqual(names(client.receive(s2, 1)), []); + assert.deepEqual(names(client.receive(s1, 2)), ["s1", "s2"]); +}); + +test("Connect seeds the watermark of the accepting side", () => { + const { client, server } = handshake(); + const first = send(client, "schema", "first", true); + const second = send(client, "schema", "second", true); + + assert.deepEqual(names(server.receive(second, 1)), []); + assert.deepEqual(names(server.receive(first, 2)), ["first", "second"]); +}); + +test("a duplicate of a held frame is not delivered twice", () => { + const { client: sender, server: receiver } = handshake(); + const s1 = send(sender, "schema", "s1", true); + const s2 = send(sender, "schema", "s2", true); + + assert.deepEqual(names(receiver.receive(s2, 1)), []); + assert.deepEqual(names(receiver.receive(s2, 2)), []); + assert.deepEqual(names(receiver.receive(s1, 3)), ["s1", "s2"]); + assert.deepEqual(names(receiver.receive(s2, 4)), []); +}); + +test("resetting peer state forgets held frames and reseeds the watermark from the next handshake", () => { + const { client: sender, server: receiver } = handshake(); + send(sender, "schema", "s1", true); + const s2 = send(sender, "schema", "s2", true); + assert.deepEqual(names(receiver.receive(s2, 1)), []); + + receiver.resetPeerState(); + const next = new CultNetRudpSession({ connectionId: 410, initialSequence: 100 }); + next.receive(receiver.acceptConnect(next.createConnect(0), 2), 2); + const d1 = send(next, "schema", "d1", true); + const d2 = send(next, "schema", "d2", true); + + // The held s2 died with the reset, and the watermark follows the new peer. + assert.deepEqual(names(receiver.receive(d2, 3)), []); + assert.deepEqual(names(receiver.receive(d1, 4)), ["d1", "d2"]); +}); diff --git a/packages/cultnet-ts/test/rudp-session-ending.test.ts b/packages/cultnet-ts/test/rudp-session-ending.test.ts new file mode 100644 index 00000000..36381023 --- /dev/null +++ b/packages/cultnet-ts/test/rudp-session-ending.test.ts @@ -0,0 +1,72 @@ +// A write belongs to the session generation it was issued in. Every way a +// generation ends drops the writes it still owed, so none is retransmitted +// into a later session where the new peer would deliver it. +import assert from "node:assert/strict"; +import test from "node:test"; +import { CultNetRudpSession } from "../src"; + +const connectionId = 0x10203070; +const enc = (text: string) => new TextEncoder().encode(text); + +function clientWithALostWrite(): { client: CultNetRudpSession; server: CultNetRudpSession } { + const client = new CultNetRudpSession({ connectionId, initialSequence: 1 }); + const server = new CultNetRudpSession({ connectionId, initialSequence: 500 }); + client.receive(server.acceptConnect(client.createConnect(0), 0), 0); + client.send("schema", enc("owed to the old session"), { reliable: true, ordered: true }); + // A fragmented write larger than the send window leaves part of it queued. + client.sendMany("schema", new Uint8Array(40 * 8), { reliable: true, ordered: true, maxFragmentBytes: 8 }); + assert.equal(client.outstandingReliablePacketCount, 41); + assert.ok(client.queuedReliablePacketCount > 0); + return { client, server }; +} + +const endings: Record void> = { + "peer Disconnect": (client, server) => { + client.receive(server.createDisconnect(enc("bye")), 1); + }, + "local disconnect": (client) => { + client.createDisconnect(enc("bye")); + }, + timeout: (client) => { + assert.equal(client.checkTimeout(1_000, 10), true); + }, + reset: (client) => { + client.resetPeerState(); + }, +}; + +for (const [name, end] of Object.entries(endings)) { + test(`a ${name} drops the writes the session owed`, () => { + const { client, server } = clientWithALostWrite(); + end(client, server); + + assert.equal(client.outstandingReliablePacketCount, 0, "the write survived the end"); + + const nextServer = new CultNetRudpSession({ connectionId, initialSequence: 900 }); + nextServer.acceptConnect(client.createConnect(2_000), 2_000); + for (const resend of client.dueResends(60_000)) { + assert.deepEqual( + nextServer.receive(resend, 60_000).delivered, + [], + "the old session's write was delivered in the next session", + ); + } + }); +} + +test("an ending does not forget what was received from the peer", () => { + const client = new CultNetRudpSession({ connectionId, initialSequence: 1 }); + const server = new CultNetRudpSession({ connectionId, initialSequence: 500 }); + client.receive(server.acceptConnect(client.createConnect(0), 0), 0); + const s1 = client.send("schema", enc("s1"), { reliable: true, ordered: true }); + const s2 = client.send("schema", enc("s2"), { reliable: true, ordered: true }); + assert.deepEqual(server.receive(s2, 1).delivered, [], "s1 is missing, so s2 is held"); + + assert.equal(server.checkTimeout(1_000, 10), true); + assert.deepEqual( + server.receive(s1, 1_001).delivered.map((frame) => new TextDecoder().decode(frame.payload)), + ["s1", "s2"], + "the ending forgot the held frame", + ); + assert.deepEqual(server.receive(s2, 1_002).delivered, [], "the ending forgot what was received"); +}); diff --git a/src/GameCult.Mesh/CultMesh.cs b/src/GameCult.Mesh/CultMesh.cs index ef41d1f7..76656699 100644 --- a/src/GameCult.Mesh/CultMesh.cs +++ b/src/GameCult.Mesh/CultMesh.cs @@ -543,8 +543,8 @@ namespace GameCult.Mesh public int BindPort { get; set; } /// Gets or sets a caller-owned bound socket. public Socket? Socket { get; set; } - /// Gets or sets the first local packet sequence. - public uint InitialSequence { get; set; } = 1; + /// Gets or sets the first local packet sequence. Unset, each session draws its own at random. + public uint? InitialSequence { get; set; } /// Gets or sets the reliable resend delay in milliseconds. public long ResendDelayMs { get; set; } = 250; /// Gets or sets the advertised transport id. diff --git a/src/GameCult.Networking/CultNetRudpSchemaServer.cs b/src/GameCult.Networking/CultNetRudpSchemaServer.cs index 41a2017d..e19fccc3 100644 --- a/src/GameCult.Networking/CultNetRudpSchemaServer.cs +++ b/src/GameCult.Networking/CultNetRudpSchemaServer.cs @@ -43,9 +43,10 @@ namespace GameCult.Networking /// public uint ConnectionId { get; set; } = 0x43554c54; /// - /// Gets or sets the first local packet sequence for each accepted peer. + /// Gets or sets the first local packet sequence for each accepted peer. Unset, each peer's + /// session draws its own at random. /// - public uint InitialSequence { get; set; } = 100; + public uint? InitialSequence { get; set; } /// /// Gets or sets the resend delay in milliseconds. /// diff --git a/src/GameCult.Networking/CultNetTransport.cs b/src/GameCult.Networking/CultNetTransport.cs index e98fdc03..9799cb38 100644 --- a/src/GameCult.Networking/CultNetTransport.cs +++ b/src/GameCult.Networking/CultNetTransport.cs @@ -768,9 +768,11 @@ namespace GameCult.Networking /// public uint ConnectionId { get; set; } /// - /// Gets or sets the first local packet sequence. + /// Gets or sets the first local packet sequence. Unset, each session draws its own at random: + /// every session is a new identity to its peer, and an options object reused for several + /// must not give them one sequence. /// - public uint InitialSequence { get; set; } = 1; + public uint? InitialSequence { get; set; } /// /// Gets or sets the resend delay in milliseconds. /// @@ -845,9 +847,11 @@ namespace GameCult.Networking /// public uint ConnectionId { get; set; } /// - /// Gets or sets the first local packet sequence. + /// Gets or sets the first local packet sequence. Unset, each session draws its own at random: + /// every session is a new identity to its peer, and an options object reused for several + /// must not give them one sequence. /// - public uint InitialSequence { get; set; } = 1; + public uint? InitialSequence { get; set; } /// /// Gets or sets the resend delay in milliseconds. /// @@ -882,18 +886,17 @@ namespace GameCult.Networking /// Maximum reliable packets admitted to the wire before acknowledgements advance the window. public const int ReliableSendWindowPackets = 32; private const int ReceivedSequenceWindow = 4096; + // How long a client keeps retransmitting a Connect nobody answered before it abandons that + // attempt for a fresh one. A Connect the server answers with an Ack, never an Accept, is one + // the server judged stale (see AcceptConnect); retransmitting the same sequence would never + // change its mind. + private const long ConnectAttemptMs = 3_000; private sealed class PendingReliablePacket { public CultNetRudpPacket Packet { get; set; } = new CultNetRudpPacket(); public long LastSentAtMs { get; set; } } - private sealed class PendingOrderedFrame - { - public CultNetRudpDeliveredFrame Frame { get; set; } = new CultNetRudpDeliveredFrame(); - public uint NextSequence { get; set; } - } - private sealed class FragmentBuffer { public string ChannelId { get; set; } = string.Empty; @@ -908,6 +911,24 @@ namespace GameCult.Networking private ushort _nextFragmentId = 1; private readonly int? _maxPendingReliablePackets; private volatile bool _connected; + // Advances every time a generation ends. Everything issued in a generation (writes, flushes) + // belongs to it and dies with it. + private long _generation; + // True from the end of a generation until the next Connect or Accept begins one. A flush started + // in that interval has no live generation to wait on. + private bool _ended; + // The sequence of the Connect that started the current generation: sent by this side, or + // accepted from the peer. A Connect repeats exactly when the session is connected and the + // Connect carries this sequence. + private uint? _connectSequence; + // A Connect this side sent is unanswered. Only then is an Accept honoured. + private bool _awaitingAccept; + // When the unanswered Connect first went out; DueResends abandons it for a fresh attempt once + // ConnectAttemptMs has passed. + private long _connectStartedAtMs; + // The payload of the Connect this side sent, kept so a fresh attempt can always be built + // whatever became of the pending packet. + private byte[] _connectPayload = Array.Empty(); private long? _lastReceivedAtMs; private uint? _highestReceivedSequence; private readonly HashSet _receivedSequences = new HashSet(); @@ -915,9 +936,14 @@ namespace GameCult.Networking private readonly object _pendingReliableGate = new object(); private readonly Dictionary _pendingReliable = new Dictionary(); private readonly Queue _queuedReliable = new Queue(); - private readonly Dictionary _orderedNextSequenceByChannel = new Dictionary(StringComparer.Ordinal); - private readonly Dictionary> _orderedBuffers = - new Dictionary>(StringComparer.Ordinal); + // Every reliable sequence up to and including this one has been received since the peer state + // was last reset. Only the handshake seeds it: the peer's Connect on the accepting side, the + // peer's Accept on the connecting side. Reliable data that arrives before that is refused. + private uint? _receivedThrough; + // Ordered frames received but not yet deliverable, keyed by first sequence. A frame is held for + // exactly one reason: a reliable sequence below it has not arrived. + private readonly SortedDictionary _orderedHeld = + new SortedDictionary(); private readonly Dictionary _fragmentBuffers = new Dictionary(StringComparer.Ordinal); /// @@ -929,18 +955,39 @@ namespace GameCult.Networking ValidateLimits(options.InitialSequence, options.MaxPendingReliablePackets); ConnectionId = options.ConnectionId; - _nextSequence = options.InitialSequence; + _nextSequence = options.InitialSequence ?? DrawSequence(); ResendDelayMs = options.ResendDelayMs; _maxPendingReliablePackets = options.MaxPendingReliablePackets; } + /// + /// A sequence drawn from a secure source in [1, 2^31). The Connect's sequence is what tells a + /// peer whether a Connect repeats one it already accepted or starts a new session, so two + /// sessions must not share one by default. + /// + private static uint DrawSequence() + { + const uint ceiling = 1u << 31; + var bytes = new byte[4]; + using (var random = System.Security.Cryptography.RandomNumberGenerator.Create()) + random.GetBytes(bytes); + return 1u + BitConverter.ToUInt32(bytes, 0) % (ceiling - 1u); + } + + // Whether `sequence` is at or before `mark` in serial order, within the receive window. The + // compare is modular, so it holds across the wrap of the 32-bit space; a sequence further back + // than the window is the duplicate test's below-window clause, and one ahead of the mark is + // never before it. + private static bool AtOrBefore(uint sequence, uint mark) => + unchecked(mark - sequence) < ReceivedSequenceWindow; + internal static readonly byte[] RefusedPacketReason = Encoding.UTF8.GetBytes("session refused a packet"); /// /// A session cannot admit a Connect when its sequence space starts exhausted or its reliable /// queue holds nothing, so neither is a usable configuration. /// - internal static void ValidateLimits(uint initialSequence, int? maxPendingReliablePackets) + internal static void ValidateLimits(uint? initialSequence, int? maxPendingReliablePackets) { if (initialSequence == uint.MaxValue) { @@ -965,6 +1012,8 @@ namespace GameCult.Networking /// Gets whether the session has completed the connect/accept handshake. /// public bool Connected => _connected; + internal long Generation => Interlocked.Read(ref _generation); + internal bool Ended => _ended; /// /// Gets the logical time of the last received packet. /// @@ -1000,14 +1049,57 @@ namespace GameCult.Networking /// public CultNetRudpPacket CreateConnect(long nowMs = 0, byte[]? payload = null) { + // A session that has had a peer starts a new generation: nothing it learned from that + // peer describes the one this Connect reaches, and what it still owed that peer no + // longer takes room in the queue. + if (_connectSequence.HasValue) + ResetPeerState(); EnsureReliableCapacity(1); + _ended = false; var packet = CreatePacket(CultNetRudpPacketType.Connect, "control", payload ?? Array.Empty(), reliable: true, ordered: true, sequenced: false); + _connectSequence = packet.Sequence; + _awaitingAccept = true; + _connectStartedAtMs = nowMs; + _connectPayload = packet.Payload ?? Array.Empty(); TrackReliable(packet, nowMs); return packet; } /// - /// Accepts a connect packet and returns a reliable ordered accept packet. + /// Whether is a Connect this session's current generation already owns: + /// a retransmit of the Connect that started it (the session is connected and the sequence is + /// that Connect's), or a stale copy of an earlier attempt by the same client (a sequence before + /// it, within the receive window). Servers that keep one session per peer ask this to tell a + /// Connect that starts a new session from one that does not; anything else that reaches + /// starts a new generation. + /// + public bool ConnectRepeats(CultNetRudpPacket packet) + { + if (packet == null) throw new ArgumentNullException(nameof(packet)); + return packet.PacketType == CultNetRudpPacketType.Connect + && _connected + && _connectSequence.HasValue + && (_connectSequence.Value == packet.Sequence + || ConnectIsStale(packet.Sequence, _connectSequence.Value)); + } + + // A Connect that precedes the current generation's within the receive window is the client's + // earlier attempt, delayed in the network: a client that retried never sends a lower sequence + // again. Restarting on it would strand the client, which honours only the Accept for its newest + // Connect. The rule is TCP's answer to a delayed SYN (RFC 5961's challenge ACK): keep the + // connection and answer with an Ack. A restarted client whose random initial sequence lands in + // this window is answered the same way and abandons the attempt for a fresh draw + // (ConnectAttemptMs). + private static bool ConnectIsStale(uint sequence, uint current) => + sequence != current && AtOrBefore(sequence, current); + + /// + /// Answers a connect packet. A repeat of the accepted connect queues nothing, so a connect storm + /// cannot grow the reliable queue: the reply is the accept still awaiting acknowledgement, or an + /// ack once it was acknowledged. A stale copy of an earlier attempt gets the same reply and + /// changes nothing else: it is not evidence the peer is alive. Any other connect ends the + /// current generation, forgets the peer and accepts a new one with a reliable ordered accept + /// packet. /// public CultNetRudpPacket AcceptConnect(CultNetRudpPacket packet, long nowMs = 0, byte[]? payload = null) { @@ -1017,30 +1109,31 @@ namespace GameCult.Networking throw new InvalidOperationException($"Expected RUDP connect packet, got {packet.PacketType}."); } + if (ConnectRepeats(packet)) + { + if (_connectSequence == packet.Sequence) + { + ApplyAcknowledgements(packet); + RememberReceived(packet.Sequence); + _lastReceivedAtMs = nowMs; + } + return PendingAcceptForResend(nowMs) ?? CreateAck(); + } + ResetPeerState(); EnsureReliableCapacity(1); - RememberReceived(packet.Sequence); + SeedReceived(packet.Sequence); + _lastReceivedAtMs = nowMs; + _connectSequence = packet.Sequence; _connected = true; + _ended = false; var response = CreatePacket(CultNetRudpPacketType.Accept, "control", payload ?? Array.Empty(), reliable: true, ordered: true, sequenced: false); TrackReliable(response, nowMs); return response; } - /// - /// Answers a connect from a peer this session has already accepted. The packet's sequence is - /// remembered and nothing is queued, so a connect storm cannot grow the reliable queue. The reply - /// is the accept still awaiting acknowledgement, or an ack once it was acknowledged. - /// - public CultNetRudpPacket AnswerRepeatedConnect(CultNetRudpPacket packet, long nowMs = 0) + // The Accept still awaiting acknowledgement, resent. + private CultNetRudpPacket? PendingAcceptForResend(long nowMs) { - RequireConnection(packet); - if (packet.PacketType != CultNetRudpPacketType.Connect) - { - throw new InvalidOperationException($"Expected RUDP connect packet, got {packet.PacketType}."); - } - - ApplyAcknowledgements(packet); - RememberReceived(packet.Sequence); - _lastReceivedAtMs = nowMs; lock (_pendingReliableGate) { foreach (var pending in _pendingReliable.Values) @@ -1052,7 +1145,7 @@ namespace GameCult.Networking } } - return CreateAck(); + return null; } /// @@ -1145,16 +1238,29 @@ namespace GameCult.Networking public CultNetRudpReceiveResult Receive(CultNetRudpPacket packet, long nowMs = 0) { RequireConnection(packet); - ApplyAcknowledgements(packet); + // An Accept counts only while this side's Connect is unanswered and the Accept names it. + // A late or duplicate one, or one from an earlier generation, must not seed the watermark + // or revive an ended session. + var honoursAccept = packet.PacketType == CultNetRudpPacketType.Accept + && _awaitingAccept + && _connectSequence.HasValue + && PacketAcknowledges(packet, _connectSequence.Value); + if (packet.PacketType == CultNetRudpPacketType.Accept && !honoursAccept) + { + return new CultNetRudpReceiveResult(); + } + // While this side's Connect awaits its Accept, only the Accept it honours retires it: an Ack + // that names the Connect (a server's reply to a repeat or a stale copy) says the server did + // not start a session. + if (honoursAccept || !_awaitingAccept) + ApplyAcknowledgements(packet); var readyToSend = PromoteQueuedReliable(nowMs); _lastReceivedAtMs = nowMs; - var expectedSequenceIfUninitialized = _highestReceivedSequence.HasValue - ? _highestReceivedSequence.Value + 1 - : packet.Sequence; - if (packet.PacketType == CultNetRudpPacketType.Accept) + if (honoursAccept) { - RememberReceived(packet.Sequence); + _awaitingAccept = false; + SeedReceived(packet.Sequence); _connected = true; return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; } @@ -1189,7 +1295,7 @@ namespace GameCult.Networking if (packet.PacketType == CultNetRudpPacketType.Disconnect) { - _connected = false; + EndSession(); return new CultNetRudpReceiveResult { ReadyToSend = readyToSend, @@ -1203,11 +1309,15 @@ namespace GameCult.Networking return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; } - var duplicate = packet.Reliable - && (_receivedSequences.Contains(packet.Sequence) - || (_highestReceivedSequence.HasValue - && packet.Sequence < _highestReceivedSequence.Value - && _highestReceivedSequence.Value - packet.Sequence >= ReceivedSequenceWindow)); + // Reliable data before the handshake has seeded the watermark has no place in the order: + // refuse it unremembered, so it is not acknowledged and the sender retransmits it once + // the handshake is done. + if (packet.Reliable && !_receivedThrough.HasValue) + { + return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; + } + + var duplicate = packet.Reliable && WasReceived(packet.Sequence); if (packet.Reliable) { RememberReceived(packet.Sequence); @@ -1217,30 +1327,34 @@ namespace GameCult.Networking return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; } + var delivered = new List(); var reassembled = Reassemble(packet); - if (reassembled == null) + if (reassembled != null) { - return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; - } - - if (!reassembled.Ordered && packet.Sequenced) - { - var newestSequence = reassembled.NextSequence - 1; - if (_latestSequencedByChannel.TryGetValue(reassembled.Frame.ChannelId, out var latestSequence) - && newestSequence <= latestSequence) + if (reassembled.Ordered) { - return new CultNetRudpReceiveResult { ReadyToSend = readyToSend }; + _orderedHeld[reassembled.Frame.Sequence] = reassembled.Frame; + } + else if (!packet.Sequenced) + { + delivered.Add(reassembled.Frame); + } + else + { + var newestSequence = reassembled.NextSequence - 1; + if (!_latestSequencedByChannel.TryGetValue(reassembled.Frame.ChannelId, out var latestSequence) + || newestSequence > latestSequence) + { + _latestSequencedByChannel[reassembled.Frame.ChannelId] = newestSequence; + delivered.Add(reassembled.Frame); + } } - _latestSequencedByChannel[reassembled.Frame.ChannelId] = newestSequence; } - return new CultNetRudpReceiveResult - { - ReadyToSend = readyToSend, - Delivered = reassembled.Ordered - ? DeliverOrdered(reassembled.Frame, reassembled.NextSequence, expectedSequenceIfUninitialized) - : new[] { reassembled.Frame } - }; + // Any reliable packet may have advanced the watermark, a fragment or an unordered frame as + // much as an ordered one, so the drain runs after all of them. + delivered.AddRange(DrainOrdered()); + return new CultNetRudpReceiveResult { ReadyToSend = readyToSend, Delivered = delivered }; } /// @@ -1272,6 +1386,10 @@ namespace GameCult.Networking /// public CultNetRudpPacket CreateAckForReceived(uint receivedSequence) { + // What Receive refused is not acknowledged by name (the ack carries only what was + // received), so its sender retransmits it. + if (!WasReceived(receivedSequence)) + return CreateAck(); var (ack, _) = AckState(); return ack >= receivedSequence && ack - receivedSequence <= 32 ? CreateAck() @@ -1287,23 +1405,39 @@ namespace GameCult.Networking } /// - /// Forgets everything learned from the peer; sequence numbers already issued stay issued. + /// The one way a session generation ends. The session stops being connected and what it still + /// owed the peer dies with it: a write not yet acknowledged is dropped, so no later session + /// retransmits it or credits an ack to it. What was learned from the peer is not touched: the + /// peer may not know the session ended, and forgetting what it sent would let its retransmits + /// be delivered twice. /// - public void ResetPeerState() + private void EndSession() { _connected = false; - _lastReceivedAtMs = null; - _highestReceivedSequence = null; - _receivedSequences.Clear(); - _nextSequencedByChannel.Clear(); - _latestSequencedByChannel.Clear(); + _ended = true; + _awaitingAccept = false; + Interlocked.Increment(ref _generation); lock (_pendingReliableGate) { _pendingReliable.Clear(); _queuedReliable.Clear(); } - _orderedNextSequenceByChannel.Clear(); - _orderedBuffers.Clear(); + } + + /// + /// Ends the current generation and forgets everything learned from the peer; sequence numbers + /// already issued stay issued. + /// + public void ResetPeerState() + { + EndSession(); + _lastReceivedAtMs = null; + _highestReceivedSequence = null; + _receivedSequences.Clear(); + _nextSequencedByChannel.Clear(); + _latestSequencedByChannel.Clear(); + _receivedThrough = null; + _orderedHeld.Clear(); _fragmentBuffers.Clear(); } @@ -1323,7 +1457,7 @@ namespace GameCult.Networking /// public CultNetRudpPacket CreateDisconnect(byte[]? reason = null) { - _connected = false; + EndSession(); return CreatePacket(CultNetRudpPacketType.Disconnect, "control", reason ?? Array.Empty(), reliable: false, ordered: false, sequenced: false); } @@ -1340,7 +1474,7 @@ namespace GameCult.Networking { return false; } - _connected = false; + EndSession(); return true; } @@ -1349,6 +1483,9 @@ namespace GameCult.Networking /// public IReadOnlyList DueResends(long nowMs) { + var fresh = AbandonUnansweredConnect(nowMs); + if (fresh != null) + return new[] { fresh }; var due = new List(); lock (_pendingReliableGate) { @@ -1515,9 +1652,69 @@ namespace GameCult.Networking } } + private static bool PacketAcknowledges(CultNetRudpPacket packet, uint sequence) + { + if (packet.Ack == sequence) + return true; + for (var bit = 0; bit < 32; bit++) + { + if ((packet.AckMask & (1u << bit)) != 0 && packet.Ack > bit && packet.Ack - (uint)bit - 1 == sequence) + return true; + } + return false; + } + + // A Connect unanswered for ConnectAttemptMs is replaced, not retransmitted further, by a Connect + // with the same payload and the abandoned sequence plus the receive window less one. Nothing + // was ever sent in an unanswered generation, so no sequence issued so far is owed. The jump + // keeps a late copy of the abandoned Connect inside the new one's stale window, so a server + // that took the new Connect answers the copy with an Ack instead of restarting; and it leaves + // the stale window of whatever generation the server holds, unless that generation sits + // exactly at the jump, and then the next attempt leaves it. The first Connect of a session is + // the only one drawn at random. + private CultNetRudpPacket? AbandonUnansweredConnect(long nowMs) + { + if (!_awaitingAccept || nowMs - _connectStartedAtMs < ConnectAttemptMs || !_connectSequence.HasValue) + return null; + var jumped = (ulong)_connectSequence.Value + ReceivedSequenceWindow - 1; + _nextSequence = (uint)Math.Min(jumped, uint.MaxValue - 1); + return CreateConnect(nowMs, _connectPayload); + } + + // The handshake's one act on the watermark: the peer's Connect or Accept is the first sequence + // of the session, whatever else it has sent. + private void SeedReceived(uint sequence) + { + _receivedThrough = sequence; + RememberReceived(sequence); + } + + // The one duplicate test: true for a sequence in the window that was received, for one below + // the window, and for one at or before the watermark. The watermark starts at the handshake's + // seed, so a frame the peer sent in an earlier generation (every sequence it issued is below + // the Connect that began this one) is a duplicate of something already delivered and is + // acknowledged, never delivered again. + private bool WasReceived(uint sequence) + { + return _receivedSequences.Contains(sequence) + || (_highestReceivedSequence.HasValue + && sequence < _highestReceivedSequence.Value + && _highestReceivedSequence.Value - sequence >= ReceivedSequenceWindow) + || (_receivedThrough.HasValue && AtOrBefore(sequence, _receivedThrough.Value)); + } + private void RememberReceived(uint sequence) { _receivedSequences.Add(sequence); + if (_receivedThrough.HasValue) + { + var through = _receivedThrough.Value; + while (through != uint.MaxValue && _receivedSequences.Contains(through + 1)) + { + through++; + } + _receivedThrough = through; + } if (!_highestReceivedSequence.HasValue || sequence > _highestReceivedSequence.Value) { _highestReceivedSequence = sequence; @@ -1625,80 +1822,35 @@ namespace GameCult.Networking }; } - private IReadOnlyList DeliverOrdered( - CultNetRudpDeliveredFrame frame, - uint nextSequenceAfterFrame, - uint expectedSequenceIfUninitialized) - { - if (!_orderedNextSequenceByChannel.TryGetValue(frame.ChannelId, out var next)) - { - next = Math.Min(expectedSequenceIfUninitialized, frame.Sequence); - _orderedNextSequenceByChannel[frame.ChannelId] = next; - } - - while (frame.Sequence > next - && _receivedSequences.Contains(next) - && (!_orderedBuffers.TryGetValue(frame.ChannelId, out var pendingBuffer) || !pendingBuffer.ContainsKey(next))) - { - next++; - _orderedNextSequenceByChannel[frame.ChannelId] = next; - } - - if (frame.Sequence < next) - { - return Array.Empty(); - } - - if (frame.Sequence > next) - { - if (!_orderedBuffers.TryGetValue(frame.ChannelId, out var buffer)) - { - buffer = new SortedDictionary(); - _orderedBuffers[frame.ChannelId] = buffer; - } - - buffer[frame.Sequence] = new PendingOrderedFrame { Frame = frame, NextSequence = nextSequenceAfterFrame }; - return Array.Empty(); - } - - _orderedNextSequenceByChannel[frame.ChannelId] = nextSequenceAfterFrame; - return new[] { frame }.Concat(DrainOrdered(frame.ChannelId)).ToArray(); - } - - private IReadOnlyList DrainOrdered(string channelId) + /// + /// Delivers, in sequence order, every held ordered frame whose first sequence is at most one past + /// the watermark. Ordered delivery has one owner: the watermark. A frame this call does not + /// deliver waits for the sequence below it, and nothing else releases it. + /// + private IReadOnlyList DrainOrdered() { var delivered = new List(); - if (!_orderedBuffers.TryGetValue(channelId, out var buffer)) + if (!_receivedThrough.HasValue) { return delivered; } - while (_orderedNextSequenceByChannel.TryGetValue(channelId, out var next) && buffer.TryGetValue(next, out var pending)) + var through = _receivedThrough.Value; + foreach (var pair in _orderedHeld) { - buffer.Remove(next); - delivered.Add(pending.Frame); - _orderedNextSequenceByChannel[channelId] = pending.NextSequence; - SkipReceivedNonChannelSequences(channelId); + if (through != uint.MaxValue && pair.Key > through + 1) + { + break; + } + delivered.Add(pair.Value); + } + foreach (var frame in delivered) + { + _orderedHeld.Remove(frame.Sequence); } - return delivered; } - private void SkipReceivedNonChannelSequences(string channelId) - { - if (!_orderedNextSequenceByChannel.TryGetValue(channelId, out var next)) - { - return; - } - - while (_receivedSequences.Contains(next) - && (!_orderedBuffers.TryGetValue(channelId, out var buffer) || !buffer.ContainsKey(next))) - { - next++; - _orderedNextSequenceByChannel[channelId] = next; - } - } - private ushort AllocateFragmentId() { var fragmentId = _nextFragmentId; @@ -1917,18 +2069,23 @@ namespace GameCult.Networking var preserved = new List(); while (_deliveredFrames.Count > 0) preserved.Add(_deliveredFrames.Dequeue()); + // A flush belongs to the generation it started in: whatever ends that generation, and + // whatever begins after it, the writes being waited on are gone. + long generation; + lock (_sessionGate) + generation = _session.Generation; try { while (true) { - // An ended session forgot its unacknowledged writes; reporting them - // flushed would be a lie. - var ended = DisconnectReason; - if (ended != null) - throw new InvalidOperationException( - "RUDP session ended before its reliable writes were acknowledged: " + Encoding.UTF8.GetString(ended)); lock (_sessionGate) { + // An ended session forgot its unacknowledged writes; reporting them flushed + // would be a lie. + if (_session.Ended || _session.Generation != generation) + throw new InvalidOperationException( + "RUDP session ended before its reliable writes were acknowledged: " + + (DisconnectReason == null ? "no reason given" : Encoding.UTF8.GetString(DisconnectReason))); if (_session.OutstandingReliablePacketCount == 0) return; } @@ -2171,7 +2328,6 @@ namespace GameCult.Networking error.SocketErrorCode == SocketError.ConnectionAborted || error.SocketErrorCode == SocketError.Shutdown) { - DisconnectReason = Array.Empty(); return false; } @@ -2198,6 +2354,10 @@ namespace GameCult.Networking _stats.PacketsDropped++; return true; } + // A Connect from another endpoint is a new client, whatever sequence it carries: the endpoint + // moves and the Connect starts a new generation. Only a Connect from the peer's own + // endpoint can be its repeat. + var connectFromNewEndpoint = false; if (_remoteEndPoint == null) { // Only a Connect claims the endpoint of a server-mode transport. @@ -2211,8 +2371,16 @@ namespace GameCult.Networking } else if (!_remoteEndPoint.Equals(remote)) { - _stats.PacketsDropped++; - return true; + if (_mode == CultNetRudpSocketMode.Server && packet.PacketType == CultNetRudpPacketType.Connect) + { + _remoteEndPoint = remote; + connectFromNewEndpoint = true; + } + else + { + _stats.PacketsDropped++; + return true; + } } TracePacket("rx", packet, received, remote); @@ -2223,17 +2391,12 @@ namespace GameCult.Networking CultNetRudpPacket accept; try { - // A connect from the peer this transport already accepted repeats: answer it - // with the accept already owed and queue nothing. - if (_session.Connected) - { - accept = _session.AnswerRepeatedConnect(packet, NowMs()); - } - else - { - accept = _session.AcceptConnect(packet, NowMs()); - DisconnectReason = null; - } + // The session answers a repeat of the connect it accepted with the accept + // already owed, and anything else with a new generation. + if (connectFromNewEndpoint) + _session.ResetPeerState(); + accept = _session.AcceptConnect(packet, NowMs()); + DisconnectReason = null; } catch (InvalidOperationException) { @@ -2399,9 +2562,10 @@ namespace GameCult.Networking /// public uint ConnectionId { get; set; } /// - /// Gets or sets the first local packet sequence for each accepted peer session. + /// Gets or sets the first local packet sequence for each accepted peer session. Unset, each + /// session draws its own at random. /// - public uint InitialSequence { get; set; } = 100; + public uint? InitialSequence { get; set; } /// /// Gets or sets the resend delay in milliseconds. /// @@ -2493,7 +2657,7 @@ namespace GameCult.Networking private const int WireBurstPackets = 32; private readonly Socket _socket; private readonly uint _connectionId; - private readonly uint _initialSequence; + private readonly uint? _initialSequence; private readonly long _resendDelayMs; private readonly int? _maxFragmentBytes; private readonly int? _maxPendingReliablePackets; @@ -2668,25 +2832,29 @@ namespace GameCult.Networking { var connectPayload = packet.Payload ?? Array.Empty(); _peers.TryGetValue(peerKey, out var admitted); - // The same payload from an admitted peer is a repeated Connect: answer it with the - // Accept already owed and queue nothing. A different payload is a new generation. - if (admitted != null && admitted.ConnectPayload.AsSpan().SequenceEqual(connectPayload)) + // The admitted peer's session decides whether this Connect repeats the one it accepted: + // answer a repeat with the Accept already owed and queue nothing. Any other Connect is + // a new generation. + if (admitted != null) { lock (admitted.SessionGate) { - CultNetRudpPacket reply; - try + if (admitted.Session.ConnectRepeats(packet)) { - reply = admitted.Session.AnswerRepeatedConnect(packet, NowMs()); - } - catch (InvalidOperationException) - { - _stats.PacketsDropped++; + CultNetRudpPacket reply; + try + { + reply = admitted.Session.AcceptConnect(packet, NowMs(), _acceptPayload); + } + catch (InvalidOperationException) + { + _stats.PacketsDropped++; + return true; + } + SendPacket(admitted.RemoteEndPoint, reply); return true; } - SendPacket(admitted.RemoteEndPoint, reply); } - return true; } var peer = new CultNetRudpSocketServerPeer( diff --git a/tests/GameCult.Networking.InteropPeer/Program.cs b/tests/GameCult.Networking.InteropPeer/Program.cs index f07908a9..17a1ea89 100644 --- a/tests/GameCult.Networking.InteropPeer/Program.cs +++ b/tests/GameCult.Networking.InteropPeer/Program.cs @@ -873,6 +873,7 @@ static void RudpServeOnce(Dictionary options) } if (disconnectReason != null) { + transport.FlushReliable(TimeSpan.FromSeconds(2)); transport.Disconnect(disconnectReason); } PollRudpAfterSend(transport, TimeSpan.FromMilliseconds(250)); diff --git a/tests/GameCult.Networking.Tests/CultNetRudpConnectGenerationTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpConnectGenerationTests.cs new file mode 100644 index 00000000..87249dd4 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpConnectGenerationTests.cs @@ -0,0 +1,840 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using GameCult.Mesh; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// A Connect's sequence says whether it repeats the one a session accepted, and the handshake + /// alone seeds the watermark that orders delivery. + /// + public sealed class CultNetRudpConnectGenerationTests + { + private const uint ConnectionId = 0x10203090; + private static readonly CultNetRudpSendOptions Ordered = new CultNetRudpSendOptions { Reliable = true, Ordered = true }; + + private static CultNetRudpSession Session(uint initialSequence) => + new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = initialSequence }); + + private static CultNetRudpPacket Send(CultNetRudpSession session, string text, bool ordered = true, string channel = "schema") => + session.Send(channel, Encoding.UTF8.GetBytes(text), new CultNetRudpSendOptions { Reliable = true, Ordered = ordered }); + + private static string[] Names(CultNetRudpReceiveResult result) => + result.Delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)).ToArray(); + + private static (CultNetRudpPacket Connect, CultNetRudpPacket Accept) Handshake(CultNetRudpSession client, CultNetRudpSession server) + { + var connect = client.CreateConnect(0); + var accept = server.AcceptConnect(connect, 0); + client.Receive(accept, 0); + Assert.That(client.Connected && server.Connected, Is.True); + return (connect, accept); + } + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + return socket; + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + [Test] + public void AReconnectOnTheSameSessionIsNotStrandedByAWriteTheEndedSessionOwed() + { + var client = Session(1); + var server = Session(500); + Handshake(client, server); + Send(server, "lost"); + server.CreateDisconnect(Encoding.UTF8.GetBytes("gone")); + + client.Receive(server.AcceptConnect(client.CreateConnect(1), 1), 1); + Assert.That(Names(client.Receive(Send(server, "after"), 2)), Is.EqualTo(new[] { "after" })); + } + + [Test] + public void AServerThatAcceptsANewClientForgetsTheOldClientsSequences() + { + var oldClient = Session(50); + var server = Session(500); + Handshake(oldClient, server); + foreach (var name in new[] { "c1", "c2", "c3" }) + Assert.That(Names(server.Receive(Send(oldClient, name), 1)), Is.EqualTo(new[] { name })); + server.CreateDisconnect(Encoding.UTF8.GetBytes("restart")); + + var newClient = Session(2); + newClient.Receive(server.AcceptConnect(newClient.CreateConnect(2), 2), 2); + Assert.That(Names(server.Receive(Send(newClient, "fresh"), 3)), Is.EqualTo(new[] { "fresh" })); + } + + [Test] + public void ARetransmittedConnectAfterDataHasFlowedDoesNotResetTheSession() + { + var client = Session(1); + var server = Session(500); + var (connect, _) = Handshake(client, server); + var frame = Send(client, "once"); + Assert.That(Names(server.Receive(frame, 1)), Is.EqualTo(new[] { "once" })); + + Assert.That(server.ConnectRepeats(connect), Is.True); + Assert.That(server.AcceptConnect(connect, 2).PacketType, Is.EqualTo(CultNetRudpPacketType.Ack), "the Accept was already acknowledged"); + Assert.That(server.OutstandingReliablePacketCount, Is.EqualTo(0), "a repeat queued something"); + Assert.That(server.Receive(frame, 3).Delivered, Is.Empty, "a retransmitted Connect made the server forget what it had delivered"); + } + + [Test] + public void AConnectWithAnotherSequenceIsNotARepeatWhateverItCarries() + { + var server = Session(500); + var connect = Session(1).CreateConnect(0, Encoding.UTF8.GetBytes("same payload")); + server.AcceptConnect(connect, 0); + Assert.That(server.ConnectRepeats(connect), Is.True); + + Assert.That(server.ConnectRepeats(Session(9).CreateConnect(0, Encoding.UTF8.GetBytes("same payload"))), Is.False); + var notAConnect = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(connect)); + notAConnect.PacketType = CultNetRudpPacketType.Ping; + Assert.That(server.ConnectRepeats(notAConnect), Is.False); + server.CreateDisconnect(); + Assert.That(server.ConnectRepeats(connect), Is.False, "an ended session repeats nothing"); + } + + [Test] + public void AnAcceptThatDoesNotNameThePendingConnectIsIgnored() + { + var client = Session(10); + var server = Session(500); + var connect = client.CreateConnect(0); + var accept = server.AcceptConnect(connect, 0); + + var stale = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(accept)); + stale.Ack = connect.Sequence + 50; + stale.AckMask = 0; + client.Receive(stale, 1); + Assert.That(client.Connected, Is.False, "a stale Accept connected the session"); + Assert.That(client.PendingReliableSequences, Is.EqualTo(new[] { connect.Sequence })); + + var named = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(accept)); + named.Ack = connect.Sequence + 3; + named.AckMask = 1u << 2; + client.Receive(named, 2); + Assert.That(client.Connected, Is.True, "the mask names the Connect"); + } + + [Test] + public void AnAcceptAfterALocalDisconnectDoesNotReconnectTheSession() + { + var client = Session(1); + var server = Session(500); + var (_, accept) = Handshake(client, server); + client.CreateDisconnect(Encoding.UTF8.GetBytes("bye")); + Assert.That(client.Connected, Is.False); + + client.Receive(accept, 5); + Assert.That(client.Connected, Is.False, "a late Accept revived an ended session"); + } + + [Test] + public void AReconnectForgetsWhatTheOldServerSent() + { + var client = Session(1); + var oldServer = Session(500); + Handshake(client, oldServer); + foreach (var name in new[] { "d1", "d2", "d3" }) + Assert.That(Names(client.Receive(Send(oldServer, name), 1)), Is.EqualTo(new[] { name })); + + var newServer = Session(501); + client.Receive(newServer.AcceptConnect(client.CreateConnect(2), 2), 2); + Assert.That(Names(client.Receive(Send(newServer, "fresh"), 3)), Is.EqualTo(new[] { "fresh" })); + } + + [Test] + public void AnAcceptForAnAbandonedConnectDoesNotConnectTheSession() + { + var client = Session(1); + var server = Session(500); + var accept = server.AcceptConnect(client.CreateConnect(0), 0); + client.CreateDisconnect(Encoding.UTF8.GetBytes("never mind")); + + client.Receive(accept, 1); + Assert.That(client.Connected, Is.False, "an Accept revived an abandoned Connect"); + } + + [Test] + public void AFullQueueOwedToAVanishedPeerDoesNotRefuseANewGeneration() + { + static CultNetRudpSession Bounded(uint initialSequence) => new CultNetRudpSession(new CultNetRudpSessionOptions + { + ConnectionId = ConnectionId, + InitialSequence = initialSequence, + MaxPendingReliablePackets = 1 + }); + + var server = Bounded(500); + server.AcceptConnect(Session(1).CreateConnect(0), 0); + Assert.That(server.OutstandingReliablePacketCount, Is.EqualTo(1), "the Accept is owed and never acknowledged"); + var next = Session(9).CreateConnect(1); + server.AcceptConnect(next, 1); + Assert.That(server.ConnectRepeats(next), Is.True); + + var client = Bounded(1); + client.CreateConnect(0); + client.CreateConnect(1); + Assert.That(client.OutstandingReliablePacketCount, Is.EqualTo(1), "only the new Connect is owed"); + } + + [Test] + public void ADuplicateAcceptDoesNotReseedTheWatermark() + { + var client = Session(1); + var server = Session(500); + var (_, accept) = Handshake(client, server); + var s1 = Send(server, "s1"); + var s2 = Send(server, "s2"); + var forged = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(accept)); + forged.Sequence = s2.Sequence; + client.Receive(forged, 1); + Assert.That(client.Receive(s2, 2).Delivered, Is.Empty, "s1 is still missing"); + Assert.That(Names(client.Receive(s1, 3)), Is.EqualTo(new[] { "s1", "s2" })); + } + + [Test] + public void ReliableDataBeforeTheAcceptIsNeitherDeliveredNorAcknowledged() + { + var client = Session(1); + var server = Session(500); + var accept = server.AcceptConnect(client.CreateConnect(0), 0); + var a = Send(server, "A"); + var b = Send(server, "B"); + + foreach (var early in new[] { b, a }) + { + Assert.That(client.Receive(early, 1).Delivered, Is.Empty); + var ack = client.CreateAckForReceived(early.Sequence); + Assert.That((ack.Ack, ack.AckMask), Is.EqualTo((0u, 0u)), "an unhandled packet was acknowledged"); + } + client.Receive(accept, 2); + Assert.That(client.Connected, Is.True); + Assert.That(server.OutstandingReliablePacketCount, Is.EqualTo(3), "nothing was acknowledged"); + + Assert.That(Names(client.Receive(a, 3)), Is.EqualTo(new[] { "A" })); + Assert.That(Names(client.Receive(b, 3)), Is.EqualTo(new[] { "B" })); + } + + [Test] + public void OrderedFramesAfterTheAcceptAreDeliveredInOrderWhateverArrivesFirst() + { + var client = Session(1); + var server = Session(500); + Handshake(client, server); + var a = Send(server, "A"); + var b = Send(server, "B"); + Assert.That(client.Receive(b, 1).Delivered, Is.Empty); + Assert.That(Names(client.Receive(a, 2)), Is.EqualTo(new[] { "A", "B" })); + } + + [Test] + public void AnUnorderedReliableFrameIsDeliveredOnReceiptNeverHeldBehindAGap() + { + var client = Session(1); + var server = Session(500); + Handshake(client, server); + Send(server, "lost"); + var unordered = Send(server, "now", ordered: false, channel: "media"); + var ordered = Send(server, "later"); + Assert.That(Names(client.Receive(unordered, 1)), Is.EqualTo(new[] { "now" })); + Assert.That(client.Receive(ordered, 2).Delivered, Is.Empty, "the ordered frame waits for the gap"); + } + + [Test] + public void APeerDisconnectDoesNotForgetWhatThePeerSent() + { + var client = Session(1); + var server = Session(500); + Handshake(client, server); + var s1 = Send(client, "s1"); + var s2 = Send(client, "s2"); + Assert.That(server.Receive(s2, 1).Delivered, Is.Empty, "s1 is missing, so s2 is held"); + + server.Receive(client.CreateDisconnect(Encoding.UTF8.GetBytes("bye")), 2); + Assert.That(Names(server.Receive(s1, 3)), Is.EqualTo(new[] { "s1", "s2" }), "the Disconnect forgot the held frame"); + Assert.That(server.Receive(s2, 4).Delivered, Is.Empty, "the Disconnect forgot what was received"); + } + + /// + /// The options say nothing about the initial sequence; each session draws its own, so one options + /// object serves many sessions without giving them one sequence. + /// + [Test] + public void SessionsFromOneOptionsObjectEachDrawTheirOwnInitialSequence() + { + Assert.That(new CultNetRudpSessionOptions().InitialSequence, Is.Null); + Assert.That(new CultNetRudpSocketTransportOptions().InitialSequence, Is.Null); + Assert.That(new CultNetRudpSocketTransportServerOptions().InitialSequence, Is.Null); + Assert.That(new RudpCultNetSchemaServerOptions().InitialSequence, Is.Null); + Assert.That(new CultMeshRudpSocketOptions().InitialSequence, Is.Null); + + var options = new CultNetRudpSessionOptions { ConnectionId = ConnectionId }; + var firsts = Enumerable.Range(0, 64) + .Select(_ => new CultNetRudpSession(options).CreateConnect(0).Sequence) + .ToArray(); + Assert.That(firsts.Distinct().Count(), Is.GreaterThan(32), "a default session does not draw its own sequence"); + Assert.That(firsts.All(value => value >= 1 && value < (1u << 31)), Is.True, "a draw left [1, 2^31)"); + } + + /// + /// A transport built from the same options each time (a reconnect loop's factory) opens each + /// connection with its own Connect sequence. + /// + [Test] + public void AReusedOptionsObjectYieldsADifferentConnectSequenceForEachTransport() + { + using var listener = Bind(); + using var clientSocket = Bind(); + var options = new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-client", + Socket = clientSocket, + Mode = CultNetRudpSocketMode.Client, + RemoteEndPoint = listener.LocalEndPoint, + ConnectionId = ConnectionId + }; + var sequences = new HashSet(); + for (var index = 0; index < 8; index++) + { + var client = new CultNetRudpSocketTransportConnection(options); + client.Connect("hello"); + sequences.Add(Drain(listener).Last(p => p.PacketType == CultNetRudpPacketType.Connect).Sequence); + } + Assert.That(sequences, Has.Count.EqualTo(8), "two connections opened with one Connect sequence"); + } + + /// + /// A listener builds a session for every client it admits; each draws its own sequence, so no two + /// clients are accepted with the same one. + /// + [Test] + public void TheListenerAcceptsEachClientWithItsOwnSequence() + { + using var serverSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = serverSocket, + ConnectionId = ConnectionId + }); + var accepts = new HashSet(); + for (var index = 0; index < 8; index++) + { + using var peerSocket = Bind(); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(Session(1).CreateConnect(0)), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + accepts.Add(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept).Sequence); + } + Assert.That(accepts, Has.Count.EqualTo(8), "two clients were accepted with one sequence"); + } + + // Ack Cut 1d: a frame from an earlier generation is a duplicate, a delayed Connect from an + // earlier attempt is stale, a Connect from a new endpoint is a new client. + + private static bool NamesSequence(CultNetRudpPacket ack, uint sequence) => + ack.Ack == sequence + || Enumerable.Range(0, 32).Any(bit => (ack.AckMask & (1u << bit)) != 0 && ack.Ack > bit && ack.Ack - (uint)bit - 1 == sequence); + + /// + /// A frame the client's old generation already delivered, retransmitted after the client + /// reconnected, is acknowledged and not delivered into the new generation. + /// + [Test] + public void AFrameFromTheClientsEarlierGenerationIsAcknowledgedNotDeliveredAgain() + { + var client = Session(10); + var server = Session(500); + Handshake(client, server); + var old = Send(client, "old"); + Assert.That(Names(server.Receive(old, 1)), Is.EqualTo(new[] { "old" })); + + var connect = client.CreateConnect(2); + client.Receive(server.AcceptConnect(connect, 2), 2); + Assert.That(old.Sequence, Is.LessThan(connect.Sequence)); + Assert.That(server.Receive(old, 3).Delivered, Is.Empty, "an earlier generation's frame was delivered again"); + Assert.That(NamesSequence(server.CreateAckForReceived(old.Sequence), old.Sequence), Is.True, "the duplicate was not acknowledged"); + Assert.That(Names(server.Receive(Send(client, "fresh"), 4)), Is.EqualTo(new[] { "fresh" })); + } + + [Test] + public void AFrameFromTheServersEarlierGenerationIsAcknowledgedNotDeliveredAgain() + { + var client = Session(10); + var server = Session(500); + Handshake(client, server); + var old = Send(server, "old"); + Assert.That(Names(client.Receive(old, 1)), Is.EqualTo(new[] { "old" })); + + client.Receive(server.AcceptConnect(client.CreateConnect(2), 2), 2); + Assert.That(client.Receive(old, 3).Delivered, Is.Empty, "an earlier generation's frame was delivered again"); + Assert.That(NamesSequence(client.CreateAckForReceived(old.Sequence), old.Sequence), Is.True); + Assert.That(Names(client.Receive(Send(server, "fresh"), 4)), Is.EqualTo(new[] { "fresh" })); + } + + [Test] + public void ADuplicateBelowTheReceiveWindowIsAcknowledgedByName() + { + var client = Session(10); + var server = Session(500); + Handshake(client, server); + var first = Send(client, "first"); + Assert.That(Names(server.Receive(first, 1)), Is.EqualTo(new[] { "first" })); + for (var index = 0; index < 4_200; index++) + { + server.Receive(Send(client, "x"), 1); + client.Receive(server.CreateAck(), 1); + } + Assert.That(server.Receive(first, 2).Delivered, Is.Empty); + Assert.That(server.CreateAckForReceived(first.Sequence).Ack, Is.EqualTo(first.Sequence), "a duplicate below the window was not acknowledged by name"); + } + + /// + /// A Connect delayed from an earlier attempt of the client that is now connected does not + /// restart the session: the server answers with an Ack and the client's ordered frames keep + /// flowing. + /// + [Test] + public void ADelayedConnectFromAnEarlierAttemptDoesNotStrandTheClient() + { + var client = Session(10); + var server = Session(500); + var earlier = client.CreateConnect(0); + var retried = client.CreateConnect(300); + var accept = server.AcceptConnect(retried, 301); + client.Receive(accept, 302); + server.Receive(client.CreateAckForReceived(accept.Sequence), 302); + Assert.That(client.Connected, Is.True); + var a = Send(client, "a"); + Assert.That(Names(server.Receive(a, 303)), Is.EqualTo(new[] { "a" })); + client.Receive(server.CreateAckForReceived(a.Sequence), 303); + + Assert.That(server.ConnectRepeats(earlier), Is.True, "the delayed Connect starts nothing"); + var reply = server.AcceptConnect(earlier, 304); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + client.Receive(reply, 305); + Assert.That(client.Connected, Is.True); + + var b = Send(client, "b"); + Assert.That(Names(server.Receive(b, 306)), Is.EqualTo(new[] { "b" }), "the delayed Connect reset the server"); + client.Receive(server.CreateAckForReceived(b.Sequence), 307); + Assert.That(client.PendingReliableSequences, Does.Not.Contain(b.Sequence)); + } + + /// + /// A stale Connect is the peer's echo from the past, not evidence it is alive; a retransmit of + /// the accepted Connect is. + /// + [Test] + public void ARepeatedConnectRefreshesLivenessAndAStaleOneDoesNot() + { + var client = Session(10); + var server = Session(500); + var earlier = client.CreateConnect(0); + var current = client.CreateConnect(1); + server.AcceptConnect(current, 0); + + server.AcceptConnect(current, 900); + Assert.That(server.CheckTimeout(1_000, 500), Is.False, "a repeated Connect did not refresh liveness"); + + server.AcceptConnect(earlier, 1_400); + Assert.That(server.CheckTimeout(1_600, 500), Is.True, "a stale Connect refreshed liveness"); + } + + /// + /// An accepted peer that never speaks again times out: accepting a Connect is the first thing + /// heard from it. + /// + [Test] + public void AnAcceptedPeerThatGoesSilentTimesOut() + { + var server = Session(500); + server.AcceptConnect(Session(10).CreateConnect(0), 0); + Assert.That(server.CheckTimeout(100_000, 1_000), Is.True); + } + + [Test] + public void StaleConnectsAreRecognisedBySerialArithmetic() + { + var server = Session(500); + server.AcceptConnect(Session(3).CreateConnect(0), 0); + CultNetRudpPacket Connect(uint sequence) => Session(sequence).CreateConnect(0); + Assert.That(server.ConnectRepeats(Connect(uint.MaxValue - 1)), Is.True, "just before the wrap"); + Assert.That(server.ConnectRepeats(Connect(4)), Is.False, "just after"); + Assert.That(server.ConnectRepeats(Connect(3 + 4_096)), Is.False, "far after"); + Assert.That(server.ConnectRepeats(Connect(unchecked(3u - 4_095))), Is.True, "the window's edge"); + Assert.That(server.ConnectRepeats(Connect(unchecked(3u - 4_096))), Is.False, "past the window"); + } + + /// + /// A restarted client whose random first sequence lands just before the server's current + /// Connect is answered with an Ack, never an Accept. It does not retransmit that Connect for + /// ever: once the attempt times out it starts a fresh one with a newly drawn sequence, and is + /// admitted. + /// + [Test] + public void ARestartedClientWhoseFirstSequenceIsStaleConnectsAfterRedrawing() + { + var server = Session(500); + var old = Session(1_000); + var oldAccept = server.AcceptConnect(old.CreateConnect(0), 0); + old.Receive(oldAccept, 0); + + var restarted = Session(900); + var first = restarted.CreateConnect(0, Encoding.UTF8.GetBytes("join")); + // The old client has not acknowledged its Accept yet, so that is the reply: it names the + // old Connect, not this one. + var early = server.AcceptConnect(first, 1); + Assert.That(early.PacketType, Is.EqualTo(CultNetRudpPacketType.Accept)); + restarted.Receive(early, 1); + Assert.That(restarted.Connected, Is.False, "an Accept for another Connect connected the client"); + server.Receive(old.CreateAckForReceived(oldAccept.Sequence), 1); + var reply = server.AcceptConnect(first, 2); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + restarted.Receive(reply, 2); + Assert.That(restarted.Connected, Is.False); + + var retransmitted = restarted.DueResends(1_000); + Assert.That(retransmitted, Has.Count.EqualTo(1)); + Assert.That(retransmitted[0].Sequence, Is.EqualTo(first.Sequence), "the attempt is still young"); + + var fresh = restarted.DueResends(3_500); + Assert.That(fresh, Has.Count.EqualTo(1)); + Assert.That(fresh[0].PacketType, Is.EqualTo(CultNetRudpPacketType.Connect)); + Assert.That(fresh[0].Sequence, Is.Not.EqualTo(first.Sequence), "the same Connect was retransmitted for ever"); + Assert.That(Encoding.UTF8.GetString(fresh[0].Payload), Is.EqualTo("join"), "the fresh attempt lost the Connect payload"); + + var accept = server.AcceptConnect(fresh[0], 3_500); + Assert.That(accept.PacketType, Is.EqualTo(CultNetRudpPacketType.Accept)); + restarted.Receive(accept, 3_500); + Assert.That(restarted.Connected, Is.True); + Assert.That(Names(server.Receive(Send(restarted, "hello"), 3_501)), Is.EqualTo(new[] { "hello" })); + } + + [Test] + public void AnAnsweredConnectIsNeverReplacedByTheAttemptTimeout() + { + var client = Session(10); + var server = Session(500); + var (connect, _) = Handshake(client, server); + var resent = client.DueResends(60_000); + Assert.That(resent.All(p => p.PacketType != CultNetRudpPacketType.Connect || p.Sequence == connect.Sequence), Is.True, "a connected client started a new attempt"); + Assert.That(client.Connected, Is.True); + } + + /// + /// A pinned client that restarts on a new port against a server-mode transport is a new client: + /// the Connect starts a new generation and the client is admitted, though its sequence is the + /// one the old client's Connect carried. + /// + [Test] + public void ServerModeAdmitsAPinnedClientThatRestartsOnANewPort() + { + using var serverSocket = Bind(); + using var firstSocket = Bind(); + using var secondSocket = Bind(); + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = ConnectionId + }); + var serverEndPoint = serverSocket.LocalEndPoint!; + void ToServer(Socket from, CultNetRudpPacket packet) => from.SendTo(CultNetRudpPacketCodec.Encode(packet), serverEndPoint); + + var first = Session(1); + ToServer(firstSocket, first.CreateConnect(0, Encoding.UTF8.GetBytes("join"))); + server.ReceiveOnce(); + first.Receive(Drain(firstSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + ToServer(firstSocket, Send(first, "hello")); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo("hello")); + + var second = Session(1); + ToServer(secondSocket, second.CreateConnect(0, Encoding.UTF8.GetBytes("join"))); + server.ReceiveOnce(); + var accepts = Drain(secondSocket).Where(p => p.PacketType == CultNetRudpPacketType.Accept).ToArray(); + Assert.That(accepts, Is.Not.Empty, "the restarted client on a new port was not admitted"); + second.Receive(accepts[0], 0); + ToServer(secondSocket, Send(second, "after")); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo("after")); + } + + // Ack Cut 1d, batch 2: the connect-attempt timeout. + + /// + /// A server that keeps a session answers a client's repeated or stale Connect with an Ack that + /// may name the client's pending Connect. That Ack says no session started: only an Accept the + /// client honours retires its Connect, so the attempt still times out and is replaced. + /// + [Test] + public void AnAckNamingThePendingConnectDoesNotRetireIt() + { + var server = Session(500); + var old = Session(1); + var oldAccept = server.AcceptConnect(old.CreateConnect(0), 0); + old.Receive(oldAccept, 0); + server.Receive(old.CreateAckForReceived(oldAccept.Sequence), 0); + + var restarted = Session(1); + var first = restarted.CreateConnect(0, Encoding.UTF8.GetBytes("join")); + var reply = server.AcceptConnect(first, 1); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + Assert.That(NamesSequence(reply, first.Sequence), Is.True, "the Ack must name the pending Connect for this test to bite"); + restarted.Receive(reply, 1); + Assert.That(restarted.Connected, Is.False); + Assert.That(restarted.PendingReliableSequences, Is.EqualTo(new[] { first.Sequence }), "an Ack retired the Connect"); + + var fresh = restarted.DueResends(3_000); + Assert.That(fresh, Has.Count.EqualTo(1), "the client waits for ever with nothing to resend"); + Assert.That(Encoding.UTF8.GetString(fresh[0].Payload), Is.EqualTo("join")); + var accept = server.AcceptConnect(fresh[0], 3_000); + Assert.That(accept.PacketType, Is.EqualTo(CultNetRudpPacketType.Accept)); + restarted.Receive(accept, 3_000); + Assert.That(restarted.Connected, Is.True); + } + + [Test] + public void ALateCopyOfAnAbandonedConnectIsStaleOnceItsReplacementIsAccepted() + { + var server = Session(500); + var client = Session(10); + var abandoned = client.CreateConnect(0); + var fresh = client.DueResends(3_000); + Assert.That(fresh, Has.Count.EqualTo(1)); + Assert.That(fresh[0].Sequence, Is.EqualTo(abandoned.Sequence + 4_095)); + var accept = server.AcceptConnect(fresh[0], 3_001); + client.Receive(accept, 3_002); + server.Receive(client.CreateAckForReceived(accept.Sequence), 3_002); + Assert.That(client.Connected, Is.True); + var a = Send(client, "a"); + Assert.That(Names(server.Receive(a, 3_003)), Is.EqualTo(new[] { "a" })); + client.Receive(server.CreateAckForReceived(a.Sequence), 3_003); + + Assert.That(server.ConnectRepeats(abandoned), Is.True, "the late copy would restart the server"); + var reply = server.AcceptConnect(abandoned, 3_004); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + client.Receive(reply, 3_005); + Assert.That(client.Connected, Is.True); + Assert.That(Names(server.Receive(Send(client, "b"), 3_006)), Is.EqualTo(new[] { "b" })); + } + + [Test] + public void AServerSittingAtTheJumpIsLeftByTheNextAttempt() + { + var server = Session(500); + var old = Session(5_095); + var oldAccept = server.AcceptConnect(old.CreateConnect(0), 0); + old.Receive(oldAccept, 0); + server.Receive(old.CreateAckForReceived(oldAccept.Sequence), 0); + + var restarted = Session(1_000); + var first = restarted.CreateConnect(0, Encoding.UTF8.GetBytes("join")); + restarted.Receive(server.AcceptConnect(first, 1), 1); + Assert.That(restarted.Connected, Is.False); + + var second = restarted.DueResends(3_000)[0]; + Assert.That(second.Sequence, Is.EqualTo(5_095u), "the jump lands on the server's generation"); + var reply = server.AcceptConnect(second, 3_001); + Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); + restarted.Receive(reply, 3_001); + Assert.That(restarted.Connected, Is.False); + + var third = restarted.DueResends(6_000)[0]; + Assert.That(third.Sequence, Is.EqualTo(5_095u + 4_095u)); + var accept = server.AcceptConnect(third, 6_001); + Assert.That(accept.PacketType, Is.EqualTo(CultNetRudpPacketType.Accept)); + restarted.Receive(accept, 6_001); + Assert.That(restarted.Connected, Is.True); + } + + /// + /// A pinned client that restarts on the same address is a repeat by sequence, so the listener + /// answers with an Ack; the client's attempt times out, and the fresh Connect (another + /// sequence) starts a new generation the listener admits. + /// + [Test] + public void TheListenerAdmitsAPinnedClientThatRestartsOnTheSameAddress() + { + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = serverSocket, + ConnectionId = ConnectionId + }); + var serverEndPoint = serverSocket.LocalEndPoint!; + void ToServer(CultNetRudpPacket packet) => peerSocket.SendTo(CultNetRudpPacketCodec.Encode(packet), serverEndPoint); + + var first = Session(1); + ToServer(first.CreateConnect(0, Encoding.UTF8.GetBytes("join"))); + server.ReceiveOnce(); + var accept = Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept); + first.Receive(accept, 0); + ToServer(first.CreateAckForReceived(accept.Sequence)); + server.ReceiveOnce(); + Drain(peerSocket); + + var restarted = Session(1); + ToServer(restarted.CreateConnect(0, Encoding.UTF8.GetBytes("join"))); + server.ReceiveOnce(); + foreach (var packet in Drain(peerSocket)) + restarted.Receive(packet, 1); + Assert.That(restarted.Connected, Is.False); + + ToServer(restarted.DueResends(3_000)[0]); + server.ReceiveOnce(); + foreach (var packet in Drain(peerSocket)) + restarted.Receive(packet, 3_001); + Assert.That(restarted.Connected, Is.True, "the restarted pinned client was never admitted"); + } + + // Transports. + + [Test] + public void ServerModeAdmitsARestartedClientKeepsItsSessionOnARetransmitAndAdmitsANewEndpoint() + { + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var otherSocket = Bind(); + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = ConnectionId + }); + var serverEndPoint = serverSocket.LocalEndPoint!; + void ToServer(Socket from, CultNetRudpPacket packet) => from.SendTo(CultNetRudpPacketCodec.Encode(packet), serverEndPoint); + + var first = Session(50_000); + var connect = first.CreateConnect(0, Encoding.UTF8.GetBytes("same")); + ToServer(peerSocket, connect); + server.ReceiveOnce(); + first.Receive(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + var sent = new[] { "c0", "c1", "c2" }.Select(name => Send(first, name)).ToArray(); + foreach (var packet in sent) + { + ToServer(peerSocket, packet); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo(Encoding.UTF8.GetString(packet.Payload))); + } + + // A retransmitted Connect is a repeat: what was delivered stays delivered once. + ToServer(peerSocket, connect); + server.ReceiveOnce(); + ToServer(peerSocket, sent[0]); + Assert.That(server.ReceiveOnce(), Is.Null, "the retransmitted Connect reset the session"); + + // The same client restarted on the same address, connection id and payload. + Drain(peerSocket); + var second = Session(7); + ToServer(peerSocket, second.CreateConnect(0, Encoding.UTF8.GetBytes("same"))); + server.ReceiveOnce(); + second.Receive(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + ToServer(peerSocket, Send(second, "after restart")); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo("after restart")); + + // A Connect from a new endpoint claims the transport. + var third = Session(9); + ToServer(otherSocket, third.CreateConnect(0, Encoding.UTF8.GetBytes("elsewhere"))); + server.ReceiveOnce(); + third.Receive(Drain(otherSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + ToServer(otherSocket, Send(third, "from elsewhere")); + Assert.That(Encoding.UTF8.GetString(server.ReceiveOnce()!.Payload), Is.EqualTo("from elsewhere")); + } + + [Test] + public void TheListenerAdmitsARestartedClientWithTheSameAddressIdAndPayload() + { + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var server = new CultNetRudpSocketTransportServer(new CultNetRudpSocketTransportServerOptions + { + RuntimeId = "csharp-rudp-listener", + Socket = serverSocket, + ConnectionId = ConnectionId + }); + var ended = new List(); + server.PeerDisconnected += ended.Add; + var serverEndPoint = serverSocket.LocalEndPoint!; + void ToServer(CultNetRudpPacket packet) => peerSocket.SendTo(CultNetRudpPacketCodec.Encode(packet), serverEndPoint); + + var first = Session(50_000); + ToServer(first.CreateConnect(0, Encoding.UTF8.GetBytes("same"))); + server.ReceiveOnce(); + first.Receive(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + for (var index = 0; index < 3; index++) + { + ToServer(Send(first, $"c{index}")); + Assert.That(server.ReceiveOnce(), Is.Not.Null); + } + Assert.That(ended, Is.Empty); + + var second = Session(7); + ToServer(second.CreateConnect(0, Encoding.UTF8.GetBytes("same"))); + server.ReceiveOnce(); + Assert.That(ended, Has.Count.EqualTo(1), "the restarted client was taken for a retransmit"); + second.Receive(Drain(peerSocket).First(p => p.PacketType == CultNetRudpPacketType.Accept), 0); + ToServer(Send(second, "after restart")); + var frame = server.ReceiveOnce(); + Assert.That(frame, Is.Not.Null); + Assert.That(Encoding.UTF8.GetString(frame!.Frame.Payload), Is.EqualTo("after restart")); + } + + /// + /// Windows reports the ICMP port-unreachable for a datagram sent to a closed port as a connection + /// reset on the next receive. Nothing ended, so nothing may be reported. Other platforms do not + /// raise it, so the test is inconclusive there. + /// + [Test] + public void AnIcmpConnectionResetDoesNotEndTheSession() + { + Assume.That(OperatingSystem.IsWindows(), "only Windows raises an ICMP connection reset on UDP"); + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = ConnectionId + }); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(Session(1).CreateConnect(0)), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + Assert.That(server.Connected, Is.True); + + var closed = Bind(); + var closedEndPoint = closed.LocalEndPoint!; + closed.Dispose(); + serverSocket.SendTo(new byte[] { 1 }, closedEndPoint); + System.Threading.Thread.Sleep(50); + server.ReceiveOnce(); + Assert.That(server.DisconnectReason, Is.Null, "an ICMP reset claimed an ending that did not happen"); + Assert.That(server.Connected, Is.True); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetRudpOrderedDeliveryTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpOrderedDeliveryTests.cs new file mode 100644 index 00000000..aeca5bc4 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpOrderedDeliveryTests.cs @@ -0,0 +1,134 @@ +#nullable enable +using System.Collections.Generic; +using System.Linq; +using System.Text; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// Ordered delivery has one owner: the contiguous received-through watermark. A frame is held for + /// exactly one reason, a reliable sequence below it has not arrived, and it is delivered in the call + /// that fills the last such gap, whichever channel the gap belonged to. + /// + public sealed class CultNetRudpOrderedDeliveryTests + { + private static (CultNetRudpSession Client, CultNetRudpSession Server) Handshake() + { + var client = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 410, InitialSequence = 1 }); + var server = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 410, InitialSequence = 500 }); + var accept = server.AcceptConnect(client.CreateConnect(0), 0); + client.Receive(accept, 0); + return (client, server); + } + + private static CultNetRudpSendOptions Reliable(bool ordered) => new CultNetRudpSendOptions { Reliable = true, Ordered = ordered }; + + private static CultNetRudpPacket Send(CultNetRudpSession session, string channel, string payload, bool ordered) + => session.Send(channel, Encoding.UTF8.GetBytes(payload), Reliable(ordered)); + + private static string[] Names(CultNetRudpReceiveResult result) + => result.Delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)).ToArray(); + + [Test] + public void OrderedFrameWaitsForAGapFilledByAnotherChannel() + { + var (sender, receiver) = Handshake(); + var o1 = Send(sender, "schema", "o1", true); + var u = Send(sender, "rel", "u", false); + var o2 = Send(sender, "schema", "o2", true); + var o3 = Send(sender, "schema", "o3", true); + var o4 = Send(sender, "schema", "o4", true); + + Assert.That(Names(receiver.Receive(o1, 1)), Is.EqualTo(new[] { "o1" })); + Assert.That(Names(receiver.Receive(o2, 2)), Is.Empty, "u is missing"); + Assert.That(Names(receiver.Receive(o3, 3)), Is.Empty, "u is missing"); + Assert.That(Names(receiver.Receive(o4, 4)), Is.Empty, "u is missing"); + Assert.That(Names(receiver.Receive(u, 5)), Is.EqualTo(new[] { "u", "o2", "o3", "o4" })); + } + + [Test] + public void TwoOrderedChannelsReleaseEachOtherInSequenceOrder() + { + var (sender, receiver) = Handshake(); + var a1 = Send(sender, "schema", "A1", true); + var b1 = Send(sender, "other", "B1", true); + var a2 = Send(sender, "schema", "A2", true); + var a3 = Send(sender, "schema", "A3", true); + + var delivered = new List(); + delivered.AddRange(Names(receiver.Receive(a1, 1))); + delivered.AddRange(Names(receiver.Receive(a2, 2))); + delivered.AddRange(Names(receiver.Receive(a3, 3))); + Assert.That(delivered, Is.EqualTo(new[] { "A1" }), "B1 is missing, so A2 and A3 wait"); + Assert.That(Names(receiver.Receive(b1, 4)), Is.EqualTo(new[] { "B1", "A2", "A3" })); + } + + [Test] + public void ChannelFirstUsedAfterOtherTrafficLosesNothing() + { + var (sender, receiver) = Handshake(); + var c1 = Send(sender, "late", "C1", true); + var c2 = Send(sender, "late", "C2", true); + var x = Send(sender, "schema", "X", true); + + Assert.That(Names(receiver.Receive(x, 1)), Is.Empty, "C1 is missing"); + Assert.That(Names(receiver.Receive(c2, 2)), Is.Empty, "C1 is missing"); + Assert.That(Names(receiver.Receive(c1, 3)), Is.EqualTo(new[] { "C1", "C2", "X" })); + } + + [Test] + public void AcceptSeedsTheWatermarkOfTheConnectingSide() + { + var (client, server) = Handshake(); + var s1 = Send(server, "schema", "s1", true); + var s2 = Send(server, "schema", "s2", true); + + Assert.That(Names(client.Receive(s2, 1)), Is.Empty); + Assert.That(Names(client.Receive(s1, 2)), Is.EqualTo(new[] { "s1", "s2" })); + } + + [Test] + public void ConnectSeedsTheWatermarkOfTheAcceptingSide() + { + var (client, server) = Handshake(); + var first = Send(client, "schema", "first", true); + var second = Send(client, "schema", "second", true); + + Assert.That(Names(server.Receive(second, 1)), Is.Empty); + Assert.That(Names(server.Receive(first, 2)), Is.EqualTo(new[] { "first", "second" })); + } + + [Test] + public void DuplicateOfAHeldFrameIsNotDeliveredTwice() + { + var (sender, receiver) = Handshake(); + var s1 = Send(sender, "schema", "s1", true); + var s2 = Send(sender, "schema", "s2", true); + + Assert.That(Names(receiver.Receive(s2, 1)), Is.Empty); + Assert.That(Names(receiver.Receive(s2, 2)), Is.Empty); + Assert.That(Names(receiver.Receive(s1, 3)), Is.EqualTo(new[] { "s1", "s2" })); + Assert.That(Names(receiver.Receive(s2, 4)), Is.Empty); + } + + [Test] + public void ResetPeerStateForgetsHeldFramesAndReseedsTheWatermarkFromTheNextHandshake() + { + var (sender, receiver) = Handshake(); + Send(sender, "schema", "s1", true); + var s2 = Send(sender, "schema", "s2", true); + Assert.That(Names(receiver.Receive(s2, 1)), Is.Empty); + + receiver.ResetPeerState(); + var next = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 410, InitialSequence = 100 }); + next.Receive(receiver.AcceptConnect(next.CreateConnect(0), 2), 2); + var d1 = Send(next, "schema", "d1", true); + var d2 = Send(next, "schema", "d2", true); + + // The held s2 died with the reset, and the watermark follows the new peer. + Assert.That(Names(receiver.Receive(d2, 3)), Is.Empty); + Assert.That(Names(receiver.Receive(d1, 4)), Is.EqualTo(new[] { "d1", "d2" })); + } + } +} diff --git a/tests/GameCult.Networking.Tests/CultNetRudpSessionEndingTests.cs b/tests/GameCult.Networking.Tests/CultNetRudpSessionEndingTests.cs new file mode 100644 index 00000000..35315bc4 --- /dev/null +++ b/tests/GameCult.Networking.Tests/CultNetRudpSessionEndingTests.cs @@ -0,0 +1,215 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace GameCult.Networking.Tests +{ + /// + /// A write belongs to the session generation it was issued in. Every way a generation ends drops + /// the writes it still owed, and a flush belongs to the generation it started in. + /// + public sealed class CultNetRudpSessionEndingTests + { + private const uint ConnectionId = 0x10203070; + + private static Socket Bind() + { + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); + socket.ReceiveTimeout = 20; + return socket; + } + + private static (CultNetRudpSession Client, CultNetRudpSession Server) ClientWithALostWrite() + { + var client = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 1 }); + var server = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 500 }); + client.Receive(server.AcceptConnect(client.CreateConnect(0), 0), 0); + client.Send("schema", Encoding.UTF8.GetBytes("owed to the old session"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); + // A fragmented write larger than the send window leaves part of it queued. + client.SendMany("schema", new byte[40 * 8], new CultNetRudpSendOptions { Reliable = true, Ordered = true }, maxFragmentBytes: 8); + Assert.That(client.OutstandingReliablePacketCount, Is.EqualTo(41)); + Assert.That(client.QueuedReliablePacketCount, Is.GreaterThan(0)); + return (client, server); + } + + [TestCase("peer Disconnect")] + [TestCase("local disconnect")] + [TestCase("timeout")] + [TestCase("reset")] + public void EveryWayASessionEndsDropsTheWritesItOwed(string ending) + { + var (client, server) = ClientWithALostWrite(); + switch (ending) + { + case "peer Disconnect": client.Receive(server.CreateDisconnect(Encoding.UTF8.GetBytes("bye")), 1); break; + case "local disconnect": client.CreateDisconnect(Encoding.UTF8.GetBytes("bye")); break; + case "timeout": Assert.That(client.CheckTimeout(1_000, 10), Is.True); break; + case "reset": client.ResetPeerState(); break; + default: throw new ArgumentException(ending); + } + Assert.That(client.OutstandingReliablePacketCount, Is.EqualTo(0), "the write survived the end"); + + var nextServer = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 900 }); + nextServer.AcceptConnect(client.CreateConnect(2_000), 2_000); + foreach (var resend in client.DueResends(60_000)) + { + Assert.That( + nextServer.Receive(resend, 60_000).Delivered, + Is.Empty, + "the old session's write was delivered in the next session"); + } + } + + private static (CultNetRudpSocketTransportConnection Client, Socket Peer, EndPoint ClientEndPoint) ConnectedClient() + { + var peerSocket = Bind(); + var client = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-client", + Socket = Bind(), + Mode = CultNetRudpSocketMode.Client, + RemoteEndPoint = peerSocket.LocalEndPoint!, + ConnectionId = ConnectionId + }); + client.Connect("hello"); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + var received = peerSocket.ReceiveFrom(buffer, ref from); + var peer = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 500 }); + var accept = peer.AcceptConnect(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray()), 0); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(accept), from); + client.ReceiveOnce(); + Assert.That(client.Connected, Is.True); + return (client, peerSocket, from); + } + + [Test] + public void AFlushFailsAfterALocalDisconnectOrATimeout() + { + var (client, peerSocket, _) = ConnectedClient(); + using (client) + using (peerSocket) + { + client.Send("schema", Encoding.UTF8.GetBytes("owed")); + client.Disconnect(); + Assert.Throws(() => client.FlushReliable(TimeSpan.FromMilliseconds(200))); + } + + var (timedOut, secondPeer, _) = ConnectedClient(); + using (timedOut) + using (secondPeer) + { + timedOut.Send("schema", Encoding.UTF8.GetBytes("owed")); + Thread.Sleep(5); + Assert.That(timedOut.CheckTimeout(1), Is.True); + Assert.Throws(() => timedOut.FlushReliable(TimeSpan.FromMilliseconds(200))); + } + } + + [Test] + public void AFlushThatStartedBeforeTheEndFailsEvenAfterALaterConnect() + { + var (client, peerSocket, _) = ConnectedClient(); + using (client) + using (peerSocket) + { + client.Send("schema", Encoding.UTF8.GetBytes("owed")); + var flush = Task.Run(() => client.FlushReliable(TimeSpan.FromSeconds(5))); + Thread.Sleep(50); + client.Disconnect(); + client.Connect("hello again"); + + var error = Assert.Throws(() => flush.Wait(TimeSpan.FromSeconds(10)))!; + Assert.That(error.InnerException, Is.InstanceOf()); + } + } + + private static List Drain(Socket socket) + { + var packets = new List(); + var buffer = new byte[65535]; + EndPoint from = new IPEndPoint(IPAddress.Any, 0); + while (socket.Poll(20_000, SelectMode.SelectRead)) + { + var received = socket.ReceiveFrom(buffer, ref from); + packets.Add(CultNetRudpPacketCodec.Decode(buffer.AsSpan(0, received).ToArray())); + } + return packets; + } + + [Test] + public void AFlushStartedAfterAReconnectWaitsOnTheNewSession() + { + var (client, peerSocket, clientEndPoint) = ConnectedClient(); + using (client) + using (peerSocket) + { + client.Send("schema", Encoding.UTF8.GetBytes("owed to the old session")); + client.Disconnect(); + client.Connect("hello again"); + var connect = Drain(peerSocket).Last(packet => packet.PacketType == CultNetRudpPacketType.Connect); + var accept = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 700 }).AcceptConnect(connect, 0); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(accept), clientEndPoint); + client.ReceiveOnce(); + + Assert.DoesNotThrow(() => client.FlushReliable(TimeSpan.FromMilliseconds(500))); + } + } + + [Test] + public void AFlushStartedAfterTheServerAcceptsANewConnectWaitsOnTheNewSession() + { + using var serverSocket = Bind(); + using var peerSocket = Bind(); + using var server = new CultNetRudpSocketTransportConnection(new CultNetRudpSocketTransportOptions + { + RuntimeId = "csharp-rudp-server", + Socket = serverSocket, + Mode = CultNetRudpSocketMode.Server, + ConnectionId = ConnectionId + }); + var first = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(first.CreateConnect(0)), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(first.CreateDisconnect()), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + + var next = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId }); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(next.CreateConnect(0)), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + var accept = Drain(peerSocket).Last(packet => packet.PacketType == CultNetRudpPacketType.Accept); + next.Receive(accept, 0); + peerSocket.SendTo(CultNetRudpPacketCodec.Encode(next.CreateAck()), serverSocket.LocalEndPoint!); + server.ReceiveOnce(); + + Assert.DoesNotThrow(() => server.FlushReliable(TimeSpan.FromMilliseconds(500))); + } + + [Test] + public void AnEndingDoesNotForgetWhatWasReceivedFromThePeer() + { + var client = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 1 }); + var server = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = ConnectionId, InitialSequence = 500 }); + client.Receive(server.AcceptConnect(client.CreateConnect(0), 0), 0); + var ordered = new CultNetRudpSendOptions { Reliable = true, Ordered = true }; + var s1 = client.Send("schema", Encoding.UTF8.GetBytes("s1"), ordered); + var s2 = client.Send("schema", Encoding.UTF8.GetBytes("s2"), ordered); + Assert.That(server.Receive(s2, 1).Delivered, Is.Empty, "s1 is missing, so s2 is held"); + + Assert.That(server.CheckTimeout(1_000, 10), Is.True); + Assert.That( + server.Receive(s1, 1_001).Delivered.Select(frame => Encoding.UTF8.GetString(frame.Payload)).ToArray(), + Is.EqualTo(new[] { "s1", "s2" }), + "the ending forgot the held frame"); + Assert.That(server.Receive(s2, 1_002).Delivered, Is.Empty, "the ending forgot what was received"); + } + } +} diff --git a/tests/GameCult.Networking.Tests/NetworkingTests.cs b/tests/GameCult.Networking.Tests/NetworkingTests.cs index 1c849fd7..8bf59746 100644 --- a/tests/GameCult.Networking.Tests/NetworkingTests.cs +++ b/tests/GameCult.Networking.Tests/NetworkingTests.cs @@ -730,26 +730,25 @@ namespace GameCult.Networking.Tests InitialSequence = 200, ResendDelayMs = 100 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 7, Sequence = 1, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 7, Sequence = 2, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var first = sender.Send("schema", Encoding.UTF8.GetBytes("first"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var second = sender.Send("schema", Encoding.UTF8.GetBytes("second"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var third = sender.Send("schema", Encoding.UTF8.GetBytes("third"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); - Assert.That(sender.PendingReliableSequences, Is.EqualTo(new uint[] { 10, 11, 12 })); + Assert.That(sender.PendingReliableSequences, Is.EqualTo(new uint[] { 11, 12, 13 })); receiver.Receive(first); receiver.Receive(third); var ackWithGap = receiver.CreateAck(); - Assert.That(ackWithGap.Ack, Is.EqualTo(12)); - Assert.That(ackWithGap.AckMask, Is.EqualTo(0b10u | (1u << 9))); + Assert.That(ackWithGap.Ack, Is.EqualTo(13)); + Assert.That(ackWithGap.AckMask, Is.EqualTo(0b110u)); sender.Receive(ackWithGap); - Assert.That(sender.PendingReliableSequences, Is.EqualTo(new uint[] { 11 })); + Assert.That(sender.PendingReliableSequences, Is.EqualTo(new uint[] { 12 })); receiver.Receive(second); var fullAck = receiver.CreateAck(); - Assert.That(fullAck.Ack, Is.EqualTo(12)); - Assert.That(fullAck.AckMask, Is.EqualTo(0b11u | (1u << 9))); + Assert.That(fullAck.Ack, Is.EqualTo(13)); + Assert.That(fullAck.AckMask, Is.EqualTo(0b111u)); sender.Receive(fullAck); Assert.That(sender.PendingReliableSequences, Is.Empty); } @@ -763,7 +762,7 @@ namespace GameCult.Networking.Tests InitialSequence = 1, ResendDelayMs = 100 }); - session.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 99, Sequence = 50, ChannelId = "control" }); + session.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 99, InitialSequence = 900 }).AcceptConnect(session.CreateConnect(0), 0), 0); var sent = session.Send("schema", Encoding.UTF8.GetBytes("payload"), new CultNetRudpSendOptions { Reliable = true, Ordered = true, NowMs = 10 }); Assert.That(session.DueResends(90), Is.Empty); @@ -788,17 +787,14 @@ namespace GameCult.Networking.Tests ConnectionId = 991, InitialSequence = 500 }); - sender.Receive(new CultNetRudpPacket - { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 991, Sequence = 400, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket - { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 991, Sequence = 0, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var oldPacket = sender.Send( "snapshot", new byte[] { 1 }, new CultNetRudpSendOptions { Reliable = true, Ordered = true, NowMs = 0 }); receiver.Receive(oldPacket); - foreach (var sequence in Enumerable.Range(2, 40).Select(value => (uint)value)) + foreach (var sequence in Enumerable.Range(3, 40).Select(value => (uint)value)) { receiver.Receive(new CultNetRudpPacket { @@ -813,16 +809,16 @@ namespace GameCult.Networking.Tests } sender.Receive(receiver.CreateAck()); - Assert.That(sender.PendingReliableSequences.First(), Is.EqualTo(1u)); + Assert.That(sender.PendingReliableSequences.First(), Is.EqualTo(2u)); - var oldRetransmit = sender.DueResends(10).Single(packet => packet.Sequence == 1); + var oldRetransmit = sender.DueResends(10).Single(packet => packet.Sequence == 2); receiver.Receive(oldRetransmit); var exactAck = receiver.CreateAckForReceived(oldRetransmit.Sequence); Assert.That(exactAck.Ack, Is.EqualTo(oldRetransmit.Sequence)); Assert.That(exactAck.AckMask, Is.Zero); sender.Receive(exactAck); - Assert.That(sender.PendingReliableSequences, Does.Not.Contain(1u)); + Assert.That(sender.PendingReliableSequences, Does.Not.Contain(2u)); } [Test] @@ -834,13 +830,7 @@ namespace GameCult.Networking.Tests InitialSequence = 1, ResendDelayMs = 1 }); - session.Receive(new CultNetRudpPacket - { - PacketType = CultNetRudpPacketType.Accept, - ConnectionId = 100, - Sequence = 5000, - ChannelId = "control" - }); + session.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 100, InitialSequence = 900 }).AcceptConnect(session.CreateConnect(0), 0), 0); var packets = Enumerable.Range(0, CultNetRudpSession.ReliableSendWindowPackets) .Select(index => session.Send("schema", Encoding.UTF8.GetBytes(index.ToString()), new CultNetRudpSendOptions { Reliable = true, Ordered = true, NowMs = 0 })) @@ -909,7 +899,7 @@ namespace GameCult.Networking.Tests var accept = server.AcceptConnect(connect, 0); var repeat = CultNetRudpPacketCodec.Decode(CultNetRudpPacketCodec.Encode(connect)); repeat.Ack = accept.Sequence; - var reply = server.AnswerRepeatedConnect(repeat, 1); + var reply = server.AcceptConnect(repeat, 1); Assert.That(reply.PacketType, Is.EqualTo(CultNetRudpPacketType.Ack)); Assert.That(server.OutstandingReliablePacketCount, Is.EqualTo(0)); } @@ -968,13 +958,7 @@ namespace GameCult.Networking.Tests public void RudpSession_RejectsUnreliableOrderedDelivery() { var session = new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 196 }); - session.Receive(new CultNetRudpPacket - { - PacketType = CultNetRudpPacketType.Accept, - ConnectionId = 196, - Sequence = 50, - ChannelId = "control" - }); + session.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 196, InitialSequence = 900 }).AcceptConnect(session.CreateConnect(0), 0), 0); var error = Assert.Throws(() => session.Send( "schema", @@ -992,14 +976,14 @@ namespace GameCult.Networking.Tests InitialSequence = 1, MaxPendingReliablePackets = 2 }); - session.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 102, Sequence = 50, ChannelId = "control" }); + session.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 102, InitialSequence = 900 }).AcceptConnect(session.CreateConnect(0), 0), 0); session.Send("schema", Encoding.UTF8.GetBytes("first"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); session.Send("schema", Encoding.UTF8.GetBytes("second"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var error = Assert.Throws(() => session.Send("schema", Encoding.UTF8.GetBytes("third"), new CultNetRudpSendOptions { Reliable = true, Ordered = true })); Assert.That(error!.Message, Does.Contain("reliable send queue is full")); - Assert.That(session.PendingReliableSequences, Is.EqualTo(new uint[] { 1, 2 })); + Assert.That(session.PendingReliableSequences, Is.EqualTo(new uint[] { 2, 3 })); var fragmented = new CultNetRudpSession(new CultNetRudpSessionOptions { @@ -1007,7 +991,7 @@ namespace GameCult.Networking.Tests InitialSequence = 1, MaxPendingReliablePackets = 3 }); - fragmented.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 103, Sequence = 50, ChannelId = "control" }); + fragmented.Receive(new CultNetRudpSession(new CultNetRudpSessionOptions { ConnectionId = 103, InitialSequence = 900 }).AcceptConnect(fragmented.CreateConnect(0), 0), 0); error = Assert.Throws(() => fragmented.SendMany( @@ -1032,8 +1016,8 @@ namespace GameCult.Networking.Tests ConnectionId = 123, InitialSequence = 100 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 123, Sequence = 90, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 123, Sequence = 91, ChannelId = "control" }); + var accept = receiver.AcceptConnect(sender.CreateConnect(0), 0); + sender.Receive(accept); var first = sender.Send("schema", Encoding.UTF8.GetBytes("first"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var second = sender.Send("schema", Encoding.UTF8.GetBytes("second"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); @@ -1062,8 +1046,7 @@ namespace GameCult.Networking.Tests ConnectionId = 124, InitialSequence = 100 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 124, Sequence = 90, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 124, Sequence = 91, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var first = sender.Send("schema", Encoding.UTF8.GetBytes("first"), new CultNetRudpSendOptions { Reliable = true, Ordered = true }); var control = sender.CreateAck(); @@ -1091,8 +1074,7 @@ namespace GameCult.Networking.Tests ConnectionId = 456, InitialSequence = 100 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 456, Sequence = 90, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = 456, Sequence = 91, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var packets = sender.SendMany( "schema", @@ -1130,8 +1112,7 @@ namespace GameCult.Networking.Tests InitialSequence = 100, ResendDelayMs = 25 }); - sender.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = connectionId, Sequence = 0, ChannelId = "control" }); - receiver.Receive(new CultNetRudpPacket { PacketType = CultNetRudpPacketType.Accept, ConnectionId = connectionId, Sequence = 0, ChannelId = "control" }); + sender.Receive(receiver.AcceptConnect(sender.CreateConnect(0), 0), 0); var fragmentCount = CultNetRudpSession.ReliableSendWindowPackets + 17; var payload = Enumerable.Range(0, fragmentCount * 8).Select(index => (byte)(index % 251)).ToArray(); @@ -1269,6 +1250,16 @@ namespace GameCult.Networking.Tests Assert.That(Encoding.UTF8.GetString(ReceiveRudpFrame(client).Payload), Is.EqualTo("server still here")); } + /// Whether the packet's ack field and mask cover the sequence. + private static bool Acknowledges(CultNetRudpPacket packet, uint sequence) + { + if (packet.Ack == sequence) + return true; + return packet.Ack > sequence + && packet.Ack - sequence <= 32 + && (packet.AckMask & (1u << (int)(packet.Ack - sequence - 1))) != 0; + } + private static List DrainPackets(Socket socket) { var packets = new List(); @@ -1397,7 +1388,7 @@ namespace GameCult.Networking.Tests var goodbye = DrainPackets(peerSocket).Single(); Assert.That(goodbye.PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect)); Assert.That(peer.Receive(goodbye, 1).Disconnected, Is.True); - Assert.That(peer.PendingReliableSequences, Does.Contain(poison.Sequence)); + Assert.That(Acknowledges(goodbye, poison.Sequence), Is.False, "the goodbye acknowledged the refused frame"); // A retransmit is dropped, not acknowledged. peerSocket.SendTo(CultNetRudpPacketCodec.Encode(poison), serverEndPoint); @@ -1444,7 +1435,7 @@ namespace GameCult.Networking.Tests var goodbye = DrainPackets(peerSocket).Single(); Assert.That(goodbye.PacketType, Is.EqualTo(CultNetRudpPacketType.Disconnect)); Assert.That(peer.Receive(goodbye, 1).Disconnected, Is.True); - Assert.That(peer.PendingReliableSequences, Does.Contain(poison.Sequence)); + Assert.That(Acknowledges(goodbye, poison.Sequence), Is.False, "the goodbye acknowledged the refused frame"); peerSocket.SendTo(CultNetRudpPacketCodec.Encode(poison), serverEndPoint); Assert.That(server.ReceiveOnce(), Is.Null);