diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 09c31adf..3b79e030 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -5,11 +5,10 @@ use cultnet_rs::{ decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, encode_rudp_packet, is_permanent_send_error, send_error_code, }; -use std::collections::BTreeMap; -#[cfg(test)] -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::io::ErrorKind; use std::net::{SocketAddr, UdpSocket}; +use std::sync::mpsc::{Receiver, Sender, channel}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const MAX_UDP_DATAGRAM_BYTES: usize = 65_535; @@ -55,16 +54,85 @@ pub struct CultMeshRudpSnapshotQuery { } /// Caller-owned admission/persistence port for received raw documents. +/// +/// The sink answers each put through its `reply`, at once or later, and from +/// any thread. The server acknowledges a put to its publisher only once the +/// sink has accepted it: "acknowledged" means the sink accepted. Until every +/// put a session sent is answered, the server sends that session nothing new, +/// only resends of what it sent before. A refused put ends its session with a +/// refusal that acknowledges nothing. +/// +/// A closure returning `Result<()>` is a sink that answers at once. pub trait CultMeshRudpRawDocumentSink { - fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()>; + fn accept_raw_document( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + reply: CultMeshRudpPutReply, + ); } impl CultMeshRudpRawDocumentSink for F where F: FnMut(CultMeshRudpRawDocumentReceipt) -> Result<()>, { - fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { - self(receipt) + fn accept_raw_document( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + reply: CultMeshRudpPutReply, + ) { + match self(receipt) { + Ok(()) => reply.accept(), + Err(error) => reply.refuse(format!("{error:#}")), + } + } +} + +/// The reason a put is refused when its reply is dropped unanswered. +pub const UNANSWERED_PUT_REASON: &str = "the put was not answered"; + +/// A sink's answer to one put, on its way to the server's next poll. +struct PutAnswer { + put: u64, + refusal: Option, +} + +/// The answer a sink owes one put. Answer it with `accept` or `refuse`, now or +/// later, from any thread; the server tells the publisher on its next poll. +/// Dropping it unanswered refuses the put (`UNANSWERED_PUT_REASON`). An answer +/// that reaches a server whose session for the put has ended is discarded. +#[derive(Debug)] +#[must_use = "a put whose reply is dropped unanswered is refused"] +pub struct CultMeshRudpPutReply { + put: u64, + answers: Option>, +} + +impl CultMeshRudpPutReply { + /// The put is admitted: its publisher is acknowledged. + pub fn accept(mut self) { + self.answer(None); + } + + /// The put is refused. `reason` goes to the caller of `poll_once` as + /// `SinkRefused`; the publisher is sent only fixed text. + pub fn refuse(mut self, reason: impl std::fmt::Display) { + self.answer(Some(reason.to_string())); + } + + fn answer(&mut self, refusal: Option) { + if let Some(answers) = self.answers.take() { + // A server that is gone has nobody left to tell. + let _ = answers.send(PutAnswer { + put: self.put, + refusal, + }); + } + } +} + +impl Drop for CultMeshRudpPutReply { + fn drop(&mut self) { + self.answer(Some(UNANSWERED_PUT_REASON.into())); } } @@ -346,6 +414,35 @@ struct SessionEntry { created_at_monotonic_millis: u64, last_activity_monotonic_millis: u64, admitted_payload_bytes: usize, + /// Puts offered to the sink and not yet answered. + pending_puts: BTreeSet, + /// Delivered messages not yet handled, in the order they arrived: a + /// snapshot request waits for every earlier put's answer, and everything + /// behind it waits for it. + waiting: VecDeque<(u32, CultNetMessage)>, + /// Snapshot responses built, charged and not yet sent, because a put that + /// arrived after their request is unanswered: message id and payload. + responses_owed: VecDeque<(String, Vec)>, +} + +impl SessionEntry { + fn new(session: CultNetRudpSession, now: u64) -> Self { + Self { + session, + created_at_monotonic_millis: now, + last_activity_monotonic_millis: now, + admitted_payload_bytes: 0, + pending_puts: BTreeSet::new(), + waiting: VecDeque::new(), + responses_owed: VecDeque::new(), + } + } +} + +/// A put offered to the sink, awaiting its answer. +struct PendingPut { + session: CultMeshRudpSessionKey, + message_id: String, } /// A synchronous, multi-session CultNet RUDP raw-document server. @@ -361,6 +458,14 @@ pub struct CultMeshRudpDocumentServer { options: CultMeshRudpDocumentServerOptions, packets_dropped: u64, send_failures: u64, + /// Where replies send their answers, and the sender each reply clones. + answers: Receiver, + answer_sender: Sender, + /// Answers taken off the channel and not yet handled, in arrival order. + answered: VecDeque, + /// The next put id; ids are never reused in the server's life. + next_put: u64, + pending: BTreeMap, /// Peers whose every datagram fails to send, standing in for an unroutable /// or full path that a loopback peer cannot be made to have. #[cfg(test)] @@ -388,6 +493,7 @@ where anyhow!("CultMesh RUDP server requires a bound UDP socket: {error}") })?; socket.set_nonblocking(true)?; + let (answer_sender, answers) = channel(); Ok(Self { socket, sessions: BTreeMap::new(), @@ -397,6 +503,11 @@ where options, packets_dropped: 0, send_failures: 0, + answers, + answer_sender, + answered: VecDeque::new(), + next_put: 0, + pending: BTreeMap::new(), #[cfg(test)] failing_peers: BTreeSet::new(), #[cfg(test)] @@ -441,7 +552,13 @@ where /// message. The rejection's own detail, which can quote a sink's or /// source's error, stays with the caller. Local socket and server-state /// failures remain errors. + /// + /// Sinks' answers are handled first: an accepted put is acknowledged, and + /// a refused one is returned as the poll's rejection, one per poll. pub fn poll_once(&mut self) -> Result { + if let Some(rejection) = self.serve_answered()? { + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + } self.maintain()?; let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES]; let (received, remote_addr) = match self.socket.recv_from(&mut wire) { @@ -519,49 +636,44 @@ where } }; + // The only reply `receive` returns is a Pong. Built now, it would + // acknowledge a put still awaiting its answer. let mut unsendable = match result.reply { - Some(reply) => self.send_packet(key.remote_addr, &reply)?, - None => None, + Some(reply) if !self.acknowledgement_withheld(key) => { + self.send_packet(key.remote_addr, &reply)? + } + _ => None, }; if result.disconnected { - self.sessions.remove(&key); + self.remove_session(key); return Ok(CultMeshRudpPollOutcome::Handled); } + // A reliable packet is acknowledged below once nothing is withheld. An + // unreliable one is not, so if this poll releases a session that was + // withheld, the release acknowledges it. + let ack_release = !packet.reliable && self.acknowledgement_withheld(key); + let entry = self + .sessions + .get_mut(&key) + .expect("checked session must remain present"); for frame in result.delivered { if frame.channel_id != "schema" { continue; } - let message = match decode_cultnet_message_from_slice( + if let Ok(message) = decode_cultnet_message_from_slice( &frame.payload, CultNetWireContract::CultNetSchemaV0, ) { - Ok(message) => message, - Err(_) => continue, - }; - if let Some(rejection) = self.deliver_application_message( - key, - frame.sequence, - message, - now_unix, - now_monotonic, - )? { - // The only reply `receive` returns is a Pong, which delivers no - // frame, so no earlier send of this poll can precede a rejection. - // A response that could not be sent ended its session where the - // send failed, over the real error. - if !matches!( - rejection.reason, - CultMeshRudpRejectionReason::ResponseSendFailed(_) - ) { - self.end_session(key, SessionEnd::Rejected(rejection.reason.refusal_text()))?; - } - return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + entry.waiting.push_back((frame.sequence, message)); } } + if let Some(rejection) = self.serve_session(key, now_unix, now_monotonic, ack_release)? { + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + } - if packet.reliable && unsendable.is_none() { + if packet.reliable && unsendable.is_none() && !self.acknowledgement_withheld(key) { let ack = self .sessions .get_mut(&key) @@ -581,12 +693,18 @@ where let now = self.clock.now_monotonic_millis(); let idle_timeout_ms = duration_millis(self.options.session_idle_timeout); let lifetime_ms = duration_millis(self.options.session_max_lifetime); - let before = self.sessions.len(); - self.sessions.retain(|_, entry| { - now.saturating_sub(entry.last_activity_monotonic_millis) <= idle_timeout_ms - && now.saturating_sub(entry.created_at_monotonic_millis) <= lifetime_ms - }); - let expired = before - self.sessions.len(); + let expired: Vec<_> = self + .sessions + .iter() + .filter(|(_, entry)| { + now.saturating_sub(entry.last_activity_monotonic_millis) > idle_timeout_ms + || now.saturating_sub(entry.created_at_monotonic_millis) > lifetime_ms + }) + .map(|(key, _)| *key) + .collect(); + for key in &expired { + self.remove_session(*key); + } let mut resends = Vec::new(); for (key, entry) in &mut self.sessions { @@ -604,7 +722,7 @@ where } } Ok(CultMeshRudpMaintenance { - sessions_expired: expired, + sessions_expired: expired.len(), packets_resent: resends.len(), }) } @@ -625,7 +743,7 @@ where .get(&key) .is_some_and(|entry| !entry.session.connect_repeats(packet)) { - self.sessions.remove(&key); + self.remove_session(key); } let reply = match self.sessions.get_mut(&key) { Some(entry) => { @@ -651,18 +769,15 @@ where ), }); let accept = session.accept_connect(packet, now, Vec::new())?; - self.sessions.insert( - key, - SessionEntry { - session, - created_at_monotonic_millis: now, - last_activity_monotonic_millis: now, - admitted_payload_bytes: 0, - }, - ); + self.sessions.insert(key, SessionEntry::new(session, now)); accept } }; + // A repeated Connect is answered with an acknowledgement built now, + // which would acknowledge a put still awaiting its answer. + if self.acknowledgement_withheld(key) { + return Ok(true); + } if let Some(error) = self.send_packet(key.remote_addr, &reply)? { self.end_session(key, SessionEnd::Unsendable(error))?; return Ok(false); @@ -699,9 +814,9 @@ where let served = match self.snapshot_source.served_record(&document) { Ok(served) => served, Err(error) => { - return reject(CultMeshRudpRejectionReason::SnapshotSourceFailed( - format!("{error:#}"), - )); + return reject(CultMeshRudpRejectionReason::SnapshotSourceFailed(format!( + "{error:#}" + ))); } }; let alone = CultNetMessage::SnapshotResponseRaw { @@ -726,18 +841,32 @@ where max_fragment_count, }); } + let put = self.next_put; + self.next_put += 1; + self.sessions + .get_mut(&key) + .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before a put"))? + .pending_puts + .insert(put); + self.pending.insert( + put, + PendingPut { + session: key, + message_id: message_id.clone(), + }, + ); let receipt = CultMeshRudpRawDocumentReceipt { session: key, - message_id: message_id.clone(), + message_id, transport_sequence, received_at_unix_millis: now_unix, document, }; - if let Err(error) = self.sink.accept_raw_document(receipt) { - return reject(CultMeshRudpRejectionReason::SinkRefused(format!( - "{error:#}" - ))); - } + let reply = CultMeshRudpPutReply { + put, + answers: Some(self.answer_sender.clone()), + }; + self.sink.accept_raw_document(receipt, reply); } CultNetMessage::SnapshotRequest { message_id, @@ -809,55 +938,237 @@ where reason: CultMeshRudpRejectionReason::PayloadBudgetFull, })); } - let payload_bytes = payload.len(); + let withheld = self.acknowledgement_withheld(key); let entry = self .sessions .get_mut(&key) .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before response"))?; - let packets = match entry.session.send_many( - "schema", - payload, - CultNetRudpSendOptions { - reliable: true, - ordered: true, - sequenced: false, - now_ms: now, - reliable_expire_after_ms: None, - }, - Some(self.options.max_fragment_bytes), - ) { - Ok(packets) => packets, - Err(error) => { - return Ok(Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::SnapshotRequest, - message_id, - reason: CultMeshRudpRejectionReason::ResponseQueueFailed(format!( - "{error:#}" - )), - })); - } - }; entry.admitted_payload_bytes = - entry.admitted_payload_bytes.saturating_add(payload_bytes); - for packet in &packets { - if let Some(error) = self.send_packet(key.remote_addr, packet)? { - let code = send_error_code(&error); - self.end_session(key, SessionEnd::Unsendable(error))?; - return Ok(Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::SnapshotRequest, - message_id, - reason: CultMeshRudpRejectionReason::ResponseSendFailed(code), - })); - } + entry.admitted_payload_bytes.saturating_add(payload.len()); + if withheld { + entry.responses_owed.push_back((message_id, payload)); + return Ok(None); } + return self.send_response(key, message_id, payload, now); } _ => {} } Ok(None) } + /// Queues a charged snapshot response on its session and sends it. + fn send_response( + &mut self, + key: CultMeshRudpSessionKey, + message_id: String, + payload: Vec, + now: u64, + ) -> Result> { + let rejection = |message_id, reason| { + Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason, + })) + }; + let entry = self + .sessions + .get_mut(&key) + .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before response"))?; + let packets = match entry.session.send_many( + "schema", + payload, + CultNetRudpSendOptions { + reliable: true, + ordered: true, + sequenced: false, + now_ms: now, + reliable_expire_after_ms: None, + }, + Some(self.options.max_fragment_bytes), + ) { + Ok(packets) => packets, + Err(error) => { + return rejection( + message_id, + CultMeshRudpRejectionReason::ResponseQueueFailed(format!("{error:#}")), + ); + } + }; + for packet in &packets { + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + let code = send_error_code(&error); + self.end_session(key, SessionEnd::Unsendable(error))?; + return rejection( + message_id, + CultMeshRudpRejectionReason::ResponseSendFailed(code), + ); + } + } + Ok(None) + } + + /// Whether the session may be sent nothing new: a put it sent awaits its + /// answer, offered to the sink or waiting behind a snapshot request. Every + /// packet built now carries acknowledgement fields covering every sequence + /// received, so it would acknowledge that put. Resends of packets built + /// earlier carry their own fields and still go. The one owner of the rule. + fn acknowledgement_withheld(&self, key: CultMeshRudpSessionKey) -> bool { + self.sessions.get(&key).is_some_and(|entry| { + !entry.pending_puts.is_empty() + || entry + .waiting + .iter() + .any(|(_, message)| matches!(message, CultNetMessage::DocumentPutRaw { .. })) + }) + } + + /// Handles the sinks' answers that arrived since the last poll, session by + /// session, and returns the first refusal. + fn serve_answered(&mut self) -> Result> { + self.answered.extend(self.answers.try_iter()); + let now_unix = self.clock.now_unix_millis(); + let now = self.clock.now_monotonic_millis(); + while let Some(key) = self + .answered + .iter() + .find_map(|answer| self.pending.get(&answer.put).map(|put| put.session)) + { + // A session with an answer queued has a pending put: it is withheld. + if let Some(rejection) = self.serve_session(key, now_unix, now, true)? { + return Ok(Some(rejection)); + } + } + // What is left answers puts whose sessions ended. + self.answered.clear(); + Ok(None) + } + + /// Takes the session's answers and handles its waiting messages in order, + /// until it waits on the sink or has nothing left. A put is offered at + /// once, even behind a put awaiting its answer; a snapshot request waits + /// for every earlier put's answer. When the session stops being withheld, + /// the responses owed to it are sent, after an acknowledgement if + /// `ack_owed`. + fn serve_session( + &mut self, + key: CultMeshRudpSessionKey, + now_unix: u64, + now: u64, + mut ack_owed: bool, + ) -> Result> { + loop { + if let Some(rejection) = self.take_answers(key)? { + return Ok(Some(rejection)); + } + if !self.acknowledgement_withheld(key) { + if std::mem::take(&mut ack_owed) { + let Some(entry) = self.sessions.get_mut(&key) else { + return Ok(None); + }; + let ack = entry.session.create_ack(); + if let Some(error) = self.send_packet(key.remote_addr, &ack)? { + self.end_session(key, SessionEnd::Unsendable(error))?; + return Ok(None); + } + } + while let Some((message_id, payload)) = self + .sessions + .get_mut(&key) + .and_then(|entry| entry.responses_owed.pop_front()) + { + if let Some(rejection) = self.send_response(key, message_id, payload, now)? { + return self.reject(key, rejection); + } + } + } + let Some(entry) = self.sessions.get_mut(&key) else { + return Ok(None); + }; + let ready = match entry.waiting.front() { + None => false, + Some((_, CultNetMessage::SnapshotRequest { .. })) => entry.pending_puts.is_empty(), + Some(_) => true, + }; + let Some((sequence, message)) = ready.then(|| entry.waiting.pop_front()).flatten() + else { + return Ok(None); + }; + if let Some(rejection) = + self.deliver_application_message(key, sequence, message, now_unix, now)? + { + return self.reject(key, rejection); + } + } + } + + /// Takes the session's answers off the queue. An accepted put stops being + /// pending; a refused one ends the session and is returned. Answers for + /// puts whose session ended are discarded; other sessions' stay queued. + fn take_answers( + &mut self, + key: CultMeshRudpSessionKey, + ) -> Result> { + self.answered.extend(self.answers.try_iter()); + let mut index = 0; + while index < self.answered.len() { + let put = self.answered[index].put; + match self.pending.get(&put) { + Some(pending) if pending.session != key => { + index += 1; + continue; + } + Some(_) => {} + None => { + self.answered.remove(index); + continue; + } + } + let answer = self.answered.remove(index).expect("indexed answer"); + let pending = self.pending.remove(&put).expect("pending put"); + if let Some(entry) = self.sessions.get_mut(&key) { + entry.pending_puts.remove(&put); + } + if let Some(reason) = answer.refusal { + let rejection = CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::DocumentPutRaw, + message_id: pending.message_id, + reason: CultMeshRudpRejectionReason::SinkRefused(reason), + }; + return self.reject(key, rejection); + } + } + Ok(None) + } + + /// Ends the session over `rejection` and returns it. A response that could + /// not be sent ended its session where the send failed, over the real error. + fn reject( + &mut self, + key: CultMeshRudpSessionKey, + rejection: CultMeshRudpApplicationRejection, + ) -> Result> { + if !matches!( + rejection.reason, + CultMeshRudpRejectionReason::ResponseSendFailed(_) + ) { + self.end_session(key, SessionEnd::Rejected(rejection.reason.refusal_text()))?; + } + Ok(Some(rejection)) + } + + /// Removes a session and forgets its pending puts, so their answers are + /// discarded. Every removal goes through here. + fn remove_session(&mut self, key: CultMeshRudpSessionKey) -> Option { + let entry = self.sessions.remove(&key)?; + for put in &entry.pending_puts { + self.pending.remove(put); + } + Some(entry) + } + /// Ends a session before its peer is done with it, and says goodbye. It is /// the one way a session ends early: every end removes the session, sends /// what the peer is owed, then the goodbye. @@ -870,7 +1181,7 @@ where /// that can never be sent as built is not dropped: the goodbye names its /// error instead. A refusal that cannot be encoded is not sent. fn end_session(&mut self, key: CultMeshRudpSessionKey, end: SessionEnd) -> Result<()> { - let Some(mut entry) = self.sessions.remove(&key) else { + let Some(mut entry) = self.remove_session(key) else { return Ok(()); }; let (refusal, mut unsendable) = match end { @@ -1069,7 +1380,6 @@ fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { Ok(()) } - /// The one encoder for a snapshot response, shared by the snapshot path and by /// put admission so the size a put is judged by is the size it would be served at. fn encode_snapshot_response(response: &CultNetMessage) -> std::result::Result, String> { diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 30fe4678..9f793e75 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -1,9 +1,10 @@ use anyhow::Result; use cultmesh_rs::{ CultMeshRudpApplicationOperation, CultMeshRudpDocumentServer, - CultMeshRudpDocumentServerOptions, CultMeshRudpPollOutcome, CultMeshRudpRawDocumentReceipt, - CultMeshRudpRawDocumentSink, CultMeshRudpRejectionReason, CultMeshRudpServerClock, - CultMeshRudpSnapshotQuery, CultMeshRudpSnapshotSource, + CultMeshRudpDocumentServerOptions, CultMeshRudpPollOutcome, CultMeshRudpPutReply, + CultMeshRudpRawDocumentReceipt, CultMeshRudpRawDocumentSink, CultMeshRudpRejectionReason, + CultMeshRudpServerClock, CultMeshRudpSnapshotQuery, CultMeshRudpSnapshotSource, + UNANSWERED_PUT_REASON, }; use cultnet_rs::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, @@ -37,14 +38,19 @@ impl Sink { } impl CultMeshRudpRawDocumentSink for Sink { - fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + fn accept_raw_document( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + reply: CultMeshRudpPutReply, + ) { let mut state = self.0.lock().unwrap(); if state.failures_remaining > 0 { state.failures_remaining -= 1; - anyhow::bail!("injected sink failure CANARY-7f3a"); + reply.refuse("injected sink failure CANARY-7f3a"); + return; } state.receipts.push(receipt); - Ok(()) + reply.accept(); } } @@ -115,7 +121,12 @@ impl CultMeshRudpSnapshotSource for StrippingSource { &mut self, query: &CultMeshRudpSnapshotQuery, ) -> Result> { - Ok(self.inner.raw_snapshot(query)?.iter().map(stripped).collect()) + Ok(self + .inner + .raw_snapshot(query)? + .iter() + .map(stripped) + .collect()) } fn served_record( @@ -147,6 +158,9 @@ fn stripping_server( struct Clock { unix: Arc, monotonic: Arc, + /// A reply accepted at the next wall-clock read: an answer that arrives + /// from another thread while a poll runs. + accept_on_read: Arc>>, } impl Clock { @@ -154,6 +168,7 @@ impl Clock { Self { unix: Arc::new(AtomicU64::new(now)), monotonic: Arc::new(AtomicU64::new(now)), + accept_on_read: Arc::default(), } } @@ -173,6 +188,9 @@ impl Clock { impl CultMeshRudpServerClock for Clock { fn now_unix_millis(&self) -> u64 { + if let Some(reply) = self.accept_on_read.lock().unwrap().take() { + reply.accept(); + } self.unix.load(Ordering::SeqCst) } @@ -241,8 +259,8 @@ fn client_on( }) } -fn connect( - server: &mut CultMeshRudpDocumentServer, +fn connect( + server: &mut CultMeshRudpDocumentServer, clients: &mut [&mut CultNetRudpSocketTransportConnection], ) -> Result<()> { for client in &mut *clients { @@ -1240,7 +1258,13 @@ fn options_that_cannot_carry_an_empty_snapshot_response_are_refused() -> Result< )? .len(); let build = |options: CultMeshRudpDocumentServerOptions| { - server(options, Clock::new(63_000), Sink::default(), Source::default()).map(|_| ()) + server( + options, + Clock::new(63_000), + Sink::default(), + Source::default(), + ) + .map(|_| ()) }; let bytes = |max_snapshot_response_bytes| CultMeshRudpDocumentServerOptions { max_snapshot_response_bytes, @@ -1397,6 +1421,8 @@ struct RawPeer { socket: UdpSocket, target: SocketAddr, session: cultnet_rs::CultNetRudpSession, + /// The Connect that started the session, for a test to repeat. + connect: cultnet_rs::CultNetRudpPacket, } impl RawPeer { @@ -1406,26 +1432,58 @@ impl RawPeer { ) -> Result { let socket = UdpSocket::bind("127.0.0.1:0")?; socket.set_read_timeout(Some(Duration::from_millis(50)))?; - let mut peer = Self { + let target = server.local_addr()?; + let (session, connect) = Self::handshake(&socket, target, server, connection_id, 100)?; + Ok(Self { socket, - target: server.local_addr()?, - session: cultnet_rs::CultNetRudpSession::new(cultnet_rs::CultNetRudpSessionOptions { + target, + session, + connect, + }) + } + + /// Connects again from the same address and connection id, starting a + /// new generation of the session. + fn reconnect( + &mut self, + server: &mut CultMeshRudpDocumentServer, + ) -> Result<()> { + let connection_id = self.session.connection_id(); + (self.session, self.connect) = + Self::handshake(&self.socket, self.target, server, connection_id, 5_000)?; + Ok(()) + } + + fn handshake( + socket: &UdpSocket, + target: SocketAddr, + server: &mut CultMeshRudpDocumentServer, + connection_id: u32, + initial_sequence: u32, + ) -> Result<( + cultnet_rs::CultNetRudpSession, + cultnet_rs::CultNetRudpPacket, + )> { + let mut session = + cultnet_rs::CultNetRudpSession::new(cultnet_rs::CultNetRudpSessionOptions { connection_id, - initial_sequence: Some(100), + initial_sequence: Some(initial_sequence), resend_delay_ms: 10_000, max_pending_reliable_packets: Some(64), - }), - }; - let connect = peer.session.create_connect(0, Vec::new())?; - peer.send_packet(&connect)?; + }); + let connect = session.create_connect(0, Vec::new())?; + socket.send_to(&cultnet_rs::encode_rudp_packet(&connect)?, target)?; server.poll_once()?; - let accept = peer.receive().expect("the server accepts the connect"); + let mut wire = vec![0_u8; 65_535]; + let (bytes, _) = socket.recv_from(&mut wire)?; + let accept = cultnet_rs::decode_rudp_packet(&wire[..bytes])?; assert_eq!( accept.packet_type, - cultnet_rs::CultNetRudpPacketType::Accept + cultnet_rs::CultNetRudpPacketType::Accept, + "the server accepts the connect" ); - peer.session.receive(&accept, 1)?; - Ok(peer) + session.receive(&accept, 1)?; + Ok((session, connect)) } fn send_packet(&self, packet: &cultnet_rs::CultNetRudpPacket) -> Result<()> { @@ -1437,6 +1495,11 @@ impl RawPeer { /// Sends `message` reliably and in order; returns its sequences. fn send(&mut self, message: &CultNetMessage) -> Result> { let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; + self.send_on("schema", payload) + } + + /// Sends `payload` reliably and in order on `channel`; returns its sequences. + fn send_on(&mut self, channel: &str, payload: Vec) -> Result> { let options = cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered: true, @@ -1446,7 +1509,7 @@ impl RawPeer { }; let packets = self .session - .send_many("schema", payload, options, Some(1200))?; + .send_many(channel, payload, options, Some(1200))?; for packet in &packets { self.send_packet(packet)?; } @@ -1719,3 +1782,736 @@ fn a_response_that_is_too_large_to_send_is_named_emsgsize() -> Result<()> { ); Ok(()) } + +/// Every datagram waiting at `peer`, encoded as hex, one per line. A reliable +/// packet's own sequence is drawn at random per session, so it is written as +/// zero; everything else, the acknowledgement fields above all, is kept. +fn datagrams_hex(peer: &RawPeer) -> Result { + let mut lines = String::new(); + for mut packet in peer.drain() { + if packet.reliable { + packet.sequence = 0; + } + for byte in cultnet_rs::encode_rudp_packet(&packet)? { + lines.push_str(&format!("{byte:02x}")); + } + lines.push('\n'); + } + Ok(lines) +} + +/// A closure sink answers at once, and the server sends exactly the datagrams it +/// sent before a sink could answer later: a Pong, the put's acknowledgement, +/// a snapshot response and its acknowledgement, and for a refused put the +/// refusal and the goodbye. The bytes were recorded from the server before +/// the reply handle existed. +#[test] +fn a_closure_sink_sends_the_same_datagrams_as_before_replies_could_wait() -> Result<()> { + let accepting = |_: CultMeshRudpRawDocumentReceipt| -> Result<()> { Ok(()) }; + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + accepting, + Source::documents(vec![document("held", vec![1, 2, 3])]), + Clock::new(75_000), + Default::default(), + )?; + let mut peer = RawPeer::connect(&mut server, 80)?; + let mut seen = String::new(); + let ping = peer.session.create_ping(b"hi".to_vec()); + peer.send_packet(&ping)?; + server.poll_once()?; + seen += &datagrams_hex(&peer)?; + peer.send(&CultNetMessage::DocumentPutRaw { + message_id: "put".into(), + document: document("put", vec![4, 5, 6]), + })?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + seen += &datagrams_hex(&peer)?; + peer.send(&CultNetMessage::SnapshotRequest { + message_id: "read".into(), + schema_ids: None, + record_keys: None, + })?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + seen += &datagrams_hex(&peer)?; + + let refusing = |_: CultMeshRudpRawDocumentReceipt| -> Result<()> { + anyhow::bail!("injected sink failure CANARY-7f3a") + }; + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + refusing, + Source::default(), + Clock::new(76_000), + Default::default(), + )?; + let mut peer = RawPeer::connect(&mut server, 81)?; + peer.send(&CultNetMessage::DocumentPutRaw { + message_id: "put".into(), + document: document("put", vec![4, 5, 6]), + })?; + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("the closure refuses the put"); + }; + assert_eq!( + rejection.reason, + CultMeshRudpRejectionReason::SinkRefused("injected sink failure CANARY-7f3a".into()) + ); + seen += &datagrams_hex(&peer)?; + + assert_eq!( + seen, + concat!( + // Pong + "434e52300006002b00000050000000000000006400000000000000000000000000020700636f6e74726f6c6869 +", + // the put's acknowledgement + "434e52300004002b00000050000000000000006500000001000000000000000000000700636f6e74726f6c +", + // the snapshot response + "434e52300003032a00000050000000000000006600000003000000000000000000f50600736368656d6183ad", + "736368656d6156657273696f6ed92063756c746e65742e736e617073686f745f726573706f6e73655f7261", + "772e7630a96d6573736167654964a472656164a9646f63756d656e74739189a8736368656d614964ab7465", + "73742e7261772e7631a97265636f72644b6579a468656c64a873746f7265644174b4323032362d30392d30", + "335430303a30303a30305aaf7061796c6f6164456e636f64696e67ab6d6573736167657061636ba77061796c", + "6f6164c403010203af736f7572636552756e74696d654964ac72756e74696d652d68656c64ad736f757263", + "654167656e744964c0aa736f75726365526f6c65c0a474616773c0 +", + // the snapshot request's acknowledgement + "434e52300004002b00000050000000000000006600000003000000000000000000000700636f6e74726f6c +", + // the refusal, acknowledging nothing + "434e52300003002a00000051000000000000000000000000000000000000000000640600736368656d6185ad", + "736368656d6156657273696f6eb063756c746e65742e6572726f722e7630a56572726f72d9207468652063", + "6174616c6f6720726566757365642074686520646f63756d656e74ab726f7574696e6748696e74c0a4636f", + "6465c0a764657461696c73c0 +", + // the goodbye + "434e52300007002b00000051000000000000000000000000000000000000000000180700636f6e74726f6c", + "73657373696f6e20726566757365642061207061636b6574 +", + ) + ); + Ok(()) +} + +/// A sink that holds every put's reply for the test to answer. +#[derive(Clone, Default)] +struct HeldSink(Arc>); + +#[derive(Default)] +struct HeldState { + offered: Vec, + replies: Vec<(String, CultMeshRudpPutReply)>, +} + +impl CultMeshRudpRawDocumentSink for HeldSink { + fn accept_raw_document( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + reply: CultMeshRudpPutReply, + ) { + let mut state = self.0.lock().unwrap(); + state.offered.push(receipt.message_id.clone()); + state.replies.push((receipt.message_id, reply)); + } +} + +impl HeldSink { + /// The message ids of the puts offered so far, in order. + fn offered(&self) -> Vec { + self.0.lock().unwrap().offered.clone() + } + + /// The held reply for the put `message_id`. + fn reply(&self, message_id: &str) -> CultMeshRudpPutReply { + let mut state = self.0.lock().unwrap(); + let index = state + .replies + .iter() + .position(|(id, _)| id == message_id) + .expect("the put was offered and is unanswered"); + state.replies.remove(index).1 + } +} + +type HeldServer = CultMeshRudpDocumentServer; + +fn held_server(sink: &HeldSink, clock: &Clock) -> Result { + CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + sink.clone(), + Source::documents(vec![document("held", vec![1, 2, 3])]), + clock.clone(), + Default::default(), + ) +} + +fn put(message_id: &str) -> CultNetMessage { + CultNetMessage::DocumentPutRaw { + message_id: message_id.into(), + document: document(message_id, vec![4, 5, 6]), + } +} + +fn snapshot_request(message_id: &str) -> CultNetMessage { + CultNetMessage::SnapshotRequest { + message_id: message_id.into(), + schema_ids: None, + record_keys: None, + } +} + +/// Polls until the server has nothing left to read. +fn poll_until_idle( + server: &mut CultMeshRudpDocumentServer, +) -> Result<()> { + for _ in 0..100 { + match server.poll_once()? { + CultMeshRudpPollOutcome::Idle => return Ok(()), + CultMeshRudpPollOutcome::Handled => {} + rejected => panic!("polling must not reject: {rejected:?}"), + } + } + anyhow::bail!("the server never went idle") +} + +/// The CultNet message a single-packet schema frame carries. +fn schema_message(packet: &cultnet_rs::CultNetRudpPacket) -> Result { + assert_eq!(packet.fragment_count, 0, "fixture: one packet per message"); + decode_cultnet_message_from_slice(&packet.payload, CultNetWireContract::CultNetSchemaV0) +} + +fn assert_acknowledges_none(packets: &[cultnet_rs::CultNetRudpPacket], sequences: &[u32]) { + for packet in packets { + assert!( + sequences + .iter() + .all(|sequence| !acknowledges(packet, *sequence)), + "{:?} acknowledges an unanswered put: ack {} mask {:#x}", + packet.packet_type, + packet.ack, + packet.ack_mask + ); + } +} + +/// While a put awaits its answer, its session is sent nothing new: no +/// acknowledgement, no Pong, no snapshot response, across five resend +/// intervals. A response built before the put still resends, carrying the +/// acknowledgement fields it was built with. Once the put is accepted, the +/// next poll acknowledges it, then answers the snapshot request that waited. +#[test] +fn a_held_put_is_acknowledged_only_after_it_is_accepted() -> Result<()> { + let clock = Clock::new(80_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 90)?; + + // A response the peer never acknowledges, built before the put. + peer.send(&snapshot_request("before"))?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + let earlier: Vec = peer + .drain() + .iter() + .filter(|packet| packet.packet_type == cultnet_rs::CultNetRudpPacketType::Data) + .map(|packet| packet.sequence) + .collect(); + assert_eq!( + earlier.len(), + 1, + "fixture: the earlier response is one packet" + ); + + let held = peer.send(&put("held"))?; + poll_until_idle(&mut server)?; + assert_eq!(sink.offered(), vec!["held".to_string()]); + let mut seen = peer.drain(); + for step in 1..=5 { + clock.set(80_000 + 50 * step); + let ping = peer.session.create_ping(b"alive".to_vec()); + peer.send_packet(&ping)?; + if step == 3 { + peer.send(&snapshot_request("after"))?; + } + poll_until_idle(&mut server)?; + seen.extend(peer.drain()); + } + assert_acknowledges_none(&seen, &held); + let kinds: Vec<_> = seen + .iter() + .map(|packet| (packet.packet_type, packet.sequence)) + .collect(); + assert_eq!( + kinds, + vec![(cultnet_rs::CultNetRudpPacketType::Data, earlier[0]); 5], + "only the earlier response's resends reach the peer" + ); + + sink.reply("held").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + let released = peer.drain(); + assert_eq!(released.len(), 2, "{released:?}"); + assert_eq!( + released[0].packet_type, + cultnet_rs::CultNetRudpPacketType::Ack + ); + assert!( + held.iter() + .all(|sequence| acknowledges(&released[0], *sequence)) + ); + let CultNetMessage::SnapshotResponseRaw { message_id, .. } = schema_message(&released[1])? + else { + panic!( + "the waiting snapshot request is answered: {:?}", + released[1] + ); + }; + assert_eq!(message_id, "after"); + Ok(()) +} + +/// A put awaiting its answer holds only its own session: another session's +/// snapshot request is answered in the same polls. +#[test] +fn another_session_is_served_while_a_put_is_held() -> Result<()> { + let clock = Clock::new(81_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut holding = RawPeer::connect(&mut server, 91)?; + let mut reading = RawPeer::connect(&mut server, 92)?; + let held = holding.send(&put("held"))?; + poll_until_idle(&mut server)?; + let read = reading.send(&snapshot_request("read"))?; + poll_until_idle(&mut server)?; + + let answered = reading.drain(); + let response = answered + .iter() + .find(|packet| packet.packet_type == cultnet_rs::CultNetRudpPacketType::Data) + .expect("the other session is answered"); + assert!(matches!( + schema_message(response)?, + CultNetMessage::SnapshotResponseRaw { .. } + )); + assert!(acknowledges(response, read[0])); + assert_acknowledges_none(&holding.drain(), &held); + Ok(()) +} + +/// A refused reply refuses the put: the poll returns the sink's reason, and +/// the publisher gets the fixed refusal and a goodbye, and never an +/// acknowledgement. +#[test] +fn a_refused_reply_refuses_the_put_and_acknowledges_nothing() -> Result<()> { + let clock = Clock::new(82_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut publisher = client(server.local_addr()?, 93)?; + connect(&mut server, &mut [&mut publisher])?; + let receipt = send_reliable(&mut publisher, &put("refused"))?; + for _ in 0..200 { + publisher.receive_once()?; + publisher.poll_resends()?; + server.poll_once()?; + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(sink.offered(), vec!["refused".to_string()]); + assert_eq!( + publisher.reliable_send_status(&receipt), + CultNetRudpReliableSendStatus::Pending, + "a held put is not acknowledged, however often it is resent" + ); + + sink.reply("refused") + .refuse("the store said no CANARY-7f3a"); + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("the refusal is the next poll's outcome"); + }; + assert_eq!( + (rejection.operation, rejection.message_id, rejection.reason), + ( + CultMeshRudpApplicationOperation::DocumentPutRaw, + "refused".to_string(), + CultMeshRudpRejectionReason::SinkRefused("the store said no CANARY-7f3a".into()) + ) + ); + assert_eq!( + refusal_seen_by(&mut publisher, &receipt)?, + "the catalog refused the document" + ); + assert_eq!(server.session_count(), 0); + Ok(()) +} + +/// A reply dropped unanswered refuses its put, over a reason of its own. +#[test] +fn a_dropped_reply_refuses_the_put() -> Result<()> { + let clock = Clock::new(83_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 94)?; + let held = peer.send(&put("dropped"))?; + poll_until_idle(&mut server)?; + + drop(sink.reply("dropped")); + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("a dropped reply refuses the put"); + }; + assert_eq!( + rejection.reason, + CultMeshRudpRejectionReason::SinkRefused(UNANSWERED_PUT_REASON.into()) + ); + assert_eq!(UNANSWERED_PUT_REASON, "the put was not answered"); + let sent = peer.drain(); + assert_acknowledges_none(&sent, &held); + let kinds: Vec<_> = sent.iter().map(|packet| packet.packet_type).collect(); + assert_eq!( + kinds, + vec![ + cultnet_rs::CultNetRudpPacketType::Data, + cultnet_rs::CultNetRudpPacketType::Disconnect + ] + ); + assert_eq!( + schema_message(&sent[0])?, + CultNetMessage::Error { + error: "the catalog refused the document".into(), + code: None, + details: None, + } + ); + Ok(()) +} + +/// Two pipelined puts are both offered at once, so one write can cover them. +/// Answering the later first releases nothing; answering the earlier then +/// releases one acknowledgement covering both. A later put's acknowledgement +/// never leaves before an earlier one's. +#[test] +fn a_later_puts_answer_never_leaves_before_an_earlier_ones() -> Result<()> { + let clock = Clock::new(84_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 95)?; + let first = peer.send(&put("first"))?; + let second = peer.send(&put("second"))?; + poll_until_idle(&mut server)?; + assert_eq!(sink.offered(), vec!["first".to_string(), "second".into()]); + + sink.reply("second").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!( + peer.drain(), + Vec::new(), + "the later answer releases nothing" + ); + + sink.reply("first").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + let released = peer.drain(); + assert_eq!(released.len(), 1, "{released:?}"); + assert_eq!( + released[0].packet_type, + cultnet_rs::CultNetRudpPacketType::Ack + ); + for sequence in first.iter().chain(&second) { + assert!(acknowledges(&released[0], *sequence)); + } + Ok(()) +} + +/// A put behind a snapshot request that waits on an earlier put is not +/// offered until that request is answered, and the answer is not sent +/// while the put is unanswered: it would acknowledge the put. +#[test] +fn a_put_behind_a_waiting_snapshot_request_is_acknowledged_only_once_accepted() -> Result<()> { + let clock = Clock::new(85_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 96)?; + let first = peer.send(&put("first"))?; + peer.send(&snapshot_request("between"))?; + let second = peer.send(&put("second"))?; + poll_until_idle(&mut server)?; + assert_eq!( + sink.offered(), + vec!["first".to_string()], + "the second put waits behind the snapshot request" + ); + + sink.reply("first").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!(sink.offered(), vec!["first".to_string(), "second".into()]); + assert_eq!( + peer.drain(), + Vec::new(), + "nothing is sent while the second put is unanswered" + ); + + sink.reply("second").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + let released = peer.drain(); + assert_eq!(released.len(), 2, "{released:?}"); + assert_eq!( + released[0].packet_type, + cultnet_rs::CultNetRudpPacketType::Ack + ); + for sequence in first.iter().chain(&second) { + assert!(acknowledges(&released[0], *sequence)); + } + let CultNetMessage::SnapshotResponseRaw { message_id, .. } = schema_message(&released[1])? + else { + panic!("the snapshot request is answered: {:?}", released[1]); + }; + assert_eq!(message_id, "between"); + Ok(()) +} + +/// A repeated Connect is answered with an acknowledgement built now, so while +/// a put awaits its answer the repeat is answered with nothing. +#[test] +fn a_repeated_connect_is_not_answered_while_a_put_is_held() -> Result<()> { + let clock = Clock::new(86_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 97)?; + let held = peer.send(&put("held"))?; + poll_until_idle(&mut server)?; + peer.send_packet(&peer.connect)?; + poll_until_idle(&mut server)?; + assert_eq!(peer.drain(), Vec::new()); + + sink.reply("held").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + let released = peer.drain(); + assert_eq!(released.len(), 1, "{released:?}"); + assert!( + held.iter() + .all(|sequence| acknowledges(&released[0], *sequence)) + ); + Ok(()) +} + +/// When a session ends, the answers its puts still owe are discarded. A +/// refusal owed by an earlier generation of the same address and connection +/// id does not end the new one, and an acceptance owed by a session that +/// disconnected or expired sends nothing. +#[test] +fn a_session_that_ends_discards_the_answers_it_is_owed() -> Result<()> { + let clock = Clock::new(87_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + + // A new generation replaces the one holding the put. + let mut replaced = RawPeer::connect(&mut server, 98)?; + replaced.send(&put("replaced"))?; + poll_until_idle(&mut server)?; + replaced.reconnect(&mut server)?; + sink.reply("replaced").refuse("stale CANARY-7f3a"); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!(replaced.drain(), Vec::new()); + let ping = replaced.session.create_ping(b"new".to_vec()); + replaced.send_packet(&ping)?; + poll_until_idle(&mut server)?; + let answered = replaced.drain(); + assert_eq!( + answered + .iter() + .map(|packet| packet.packet_type) + .collect::>(), + vec![cultnet_rs::CultNetRudpPacketType::Pong], + "the new generation is served and is not withheld" + ); + + // The peer says goodbye while its put is held. + let mut departed = RawPeer::connect(&mut server, 99)?; + departed.send(&put("departed"))?; + poll_until_idle(&mut server)?; + let goodbye = departed.session.create_disconnect(Vec::new()); + departed.send_packet(&goodbye)?; + poll_until_idle(&mut server)?; + + // The session idles out while its put is held. + let mut expired = RawPeer::connect(&mut server, 100)?; + expired.send(&put("expired"))?; + poll_until_idle(&mut server)?; + clock.set(87_000 + 31_000); + poll_until_idle(&mut server)?; + assert_eq!(server.session_count(), 0); + + sink.reply("departed").accept(); + drop(sink.reply("expired")); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!(departed.drain(), Vec::new()); + assert_eq!(expired.drain(), Vec::new()); + Ok(()) +} + +/// A put the server could never serve is refused before the sink sees it. +#[test] +fn an_unservable_put_is_never_offered_to_the_sink() -> Result<()> { + let clock = Clock::new(88_000); + let sink = HeldSink::default(); + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + sink.clone(), + Source::default(), + clock, + CultMeshRudpDocumentServerOptions { + max_snapshot_response_bytes: 1_000, + ..Default::default() + }, + )?; + let mut peer = RawPeer::connect(&mut server, 101)?; + peer.send(&CultNetMessage::DocumentPutRaw { + message_id: "big".into(), + document: document("big", vec![7; 2_000]), + })?; + // The put arrives in two fragments. + let mut rejection = None; + for _ in 0..10 { + if let CultMeshRudpPollOutcome::ApplicationRejected(found) = server.poll_once()? { + rejection = Some(found); + break; + } + } + let rejection = rejection.expect("the put can never be served"); + assert!(matches!( + rejection.reason, + CultMeshRudpRejectionReason::DocumentUnservable { .. } + )); + assert_eq!(sink.offered(), Vec::::new()); + Ok(()) +} + +/// A poll whose packet finds its session released, by an answer that arrived +/// while the poll ran, sends exactly one acknowledgement. A reliable packet's +/// own acknowledgement is that one. An unreliable packet (a Ping) has none, +/// so the release sends it. +#[test] +fn a_release_inside_a_poll_sends_exactly_one_acknowledgement() -> Result<()> { + for reliable in [false, true] { + let clock = Clock::new(89_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 102)?; + let held = peer.send(&put("held"))?; + poll_until_idle(&mut server)?; + assert_eq!(peer.drain(), Vec::new(), "fixture: the put is held"); + + *clock.accept_on_read.lock().unwrap() = Some(sink.reply("held")); + let own = if reliable { + // Reliable data on a channel the server does not read. + peer.send_on("other", vec![1])? + } else { + let ping = peer.session.create_ping(Vec::new()); + peer.send_packet(&ping)?; + Vec::new() + }; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + let sent = peer.drain(); + let kinds: Vec<_> = sent.iter().map(|packet| packet.packet_type).collect(); + // No Pong: the session was withheld when the Ping arrived. + assert_eq!( + kinds, + vec![cultnet_rs::CultNetRudpPacketType::Ack], + "reliable {reliable}" + ); + for sequence in held.iter().chain(&own) { + assert!(acknowledges(&sent[0], *sequence)); + } + } + Ok(()) +} + +/// Answers are handled per session: one session's answer never releases or +/// ends another. One session's put is accepted and another's refused in the +/// same poll; each session gets only its own answer. +#[test] +fn one_sessions_answer_never_releases_or_ends_another() -> Result<()> { + let clock = Clock::new(90_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut accepted = RawPeer::connect(&mut server, 103)?; + let mut refused = RawPeer::connect(&mut server, 104)?; + let accepted_put = accepted.send(&put("accepted"))?; + let refused_put = refused.send(&put("refused"))?; + poll_until_idle(&mut server)?; + + sink.reply("accepted").accept(); + sink.reply("refused").refuse("no CANARY-7f3a"); + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("the refusal is the poll's outcome"); + }; + assert_eq!( + ( + rejection.session.connection_id, + rejection.message_id.as_str() + ), + (104, "refused") + ); + let to_accepted = accepted.drain(); + assert_eq!( + to_accepted + .iter() + .map(|packet| packet.packet_type) + .collect::>(), + vec![cultnet_rs::CultNetRudpPacketType::Ack] + ); + assert!( + accepted_put + .iter() + .all(|sequence| acknowledges(&to_accepted[0], *sequence)) + ); + let to_refused = refused.drain(); + assert_acknowledges_none(&to_refused, &refused_put); + assert_eq!( + to_refused + .iter() + .map(|packet| packet.packet_type) + .collect::>(), + vec![ + cultnet_rs::CultNetRudpPacketType::Data, + cultnet_rs::CultNetRudpPacketType::Disconnect + ] + ); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +/// A session is kept while its idle time and its age are at their limits, and +/// expires once either is past it. +#[test] +fn a_session_is_kept_at_its_limits_and_expires_past_them() -> Result<()> { + for (idle, lifetime) in [(1_000, 60_000), (60_000, 1_000)] { + let clock = Clock::new(91_000); + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + HeldSink::default(), + Source::default(), + clock.clone(), + CultMeshRudpDocumentServerOptions { + session_idle_timeout: Duration::from_millis(idle), + session_max_lifetime: Duration::from_millis(lifetime), + ..Default::default() + }, + )?; + RawPeer::connect(&mut server, 105)?; + clock.set(92_000); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!( + server.session_count(), + 1, + "kept at the limit ({idle}, {lifetime})" + ); + clock.set(92_001); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!( + server.session_count(), + 0, + "expired past it ({idle}, {lifetime})" + ); + } + Ok(()) +}