From 08e0c229c79e43c01ff39851a7e6536c47e38dd3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 00:03:08 +0200 Subject: [PATCH 1/4] cultmesh-rs: pin the datagrams a closure sink's server sends A raw peer records every datagram the document server sends for a Ping, an accepted put, a snapshot request, and a refused put, and compares their bytes with those recorded from this server. A reliable packet's own sequence is drawn at random per session, so it is written as zero; the acknowledgement fields are kept. This lands before the put reply may be deferred, so the same test shows that a sink answering at once still sends the same bytes after it. Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-rs/tests/rudp_document_server.rs | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 30fe4678..c9aaebc4 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -1719,3 +1719,115 @@ fn a_response_that_is_too_large_to_send_is_named_emsgsize() -> Result<()> { ); Ok(()) } + +/// Every datagram waiting at `peer`, encoded as hex, one per line. A reliable +/// packet's own sequence is drawn at random per session, so it is written as +/// zero; everything else, the acknowledgement fields above all, is kept. +fn datagrams_hex(peer: &RawPeer) -> Result { + let mut lines = String::new(); + for mut packet in peer.drain() { + if packet.reliable { + packet.sequence = 0; + } + for byte in cultnet_rs::encode_rudp_packet(&packet)? { + lines.push_str(&format!("{byte:02x}")); + } + lines.push('\n'); + } + Ok(lines) +} + +/// A closure sink answers at once, and the server sends exactly the datagrams it +/// sent before a sink could answer later: a Pong, the put's acknowledgement, +/// a snapshot response and its acknowledgement, and for a refused put the +/// refusal and the goodbye. The bytes were recorded from the server before +/// the reply handle existed. +#[test] +fn a_closure_sink_sends_the_same_datagrams_as_before_replies_could_wait() -> Result<()> { + let accepting = |_: CultMeshRudpRawDocumentReceipt| -> Result<()> { Ok(()) }; + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + accepting, + Source::documents(vec![document("held", vec![1, 2, 3])]), + Clock::new(75_000), + Default::default(), + )?; + let mut peer = RawPeer::connect(&mut server, 80)?; + let mut seen = String::new(); + let ping = peer.session.create_ping(b"hi".to_vec()); + peer.send_packet(&ping)?; + server.poll_once()?; + seen += &datagrams_hex(&peer)?; + peer.send(&CultNetMessage::DocumentPutRaw { + message_id: "put".into(), + document: document("put", vec![4, 5, 6]), + })?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + seen += &datagrams_hex(&peer)?; + peer.send(&CultNetMessage::SnapshotRequest { + message_id: "read".into(), + schema_ids: None, + record_keys: None, + })?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + seen += &datagrams_hex(&peer)?; + + let refusing = |_: CultMeshRudpRawDocumentReceipt| -> Result<()> { + anyhow::bail!("injected sink failure CANARY-7f3a") + }; + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + refusing, + Source::default(), + Clock::new(76_000), + Default::default(), + )?; + let mut peer = RawPeer::connect(&mut server, 81)?; + peer.send(&CultNetMessage::DocumentPutRaw { + message_id: "put".into(), + document: document("put", vec![4, 5, 6]), + })?; + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("the closure refuses the put"); + }; + assert_eq!( + rejection.reason, + CultMeshRudpRejectionReason::SinkRefused("injected sink failure CANARY-7f3a".into()) + ); + seen += &datagrams_hex(&peer)?; + + assert_eq!( + seen, + concat!( + // Pong + "434e52300006002b00000050000000000000006400000000000000000000000000020700636f6e74726f6c6869 +", + // the put's acknowledgement + "434e52300004002b00000050000000000000006500000001000000000000000000000700636f6e74726f6c +", + // the snapshot response + "434e52300003032a00000050000000000000006600000003000000000000000000f50600736368656d6183ad", + "736368656d6156657273696f6ed92063756c746e65742e736e617073686f745f726573706f6e73655f7261", + "772e7630a96d6573736167654964a472656164a9646f63756d656e74739189a8736368656d614964ab7465", + "73742e7261772e7631a97265636f72644b6579a468656c64a873746f7265644174b4323032362d30392d30", + "335430303a30303a30305aaf7061796c6f6164456e636f64696e67ab6d6573736167657061636ba77061796c", + "6f6164c403010203af736f7572636552756e74696d654964ac72756e74696d652d68656c64ad736f757263", + "654167656e744964c0aa736f75726365526f6c65c0a474616773c0 +", + // the snapshot request's acknowledgement + "434e52300004002b00000050000000000000006600000003000000000000000000000700636f6e74726f6c +", + // the refusal, acknowledging nothing + "434e52300003002a00000051000000000000000000000000000000000000000000640600736368656d6185ad", + "736368656d6156657273696f6eb063756c746e65742e6572726f722e7630a56572726f72d9207468652063", + "6174616c6f6720726566757365642074686520646f63756d656e74ab726f7574696e6748696e74c0a4636f", + "6465c0a764657461696c73c0 +", + // the goodbye + "434e52300007002b00000051000000000000000000000000000000000000000000180700636f6e74726f6c", + "73657373696f6e20726566757365642061207061636b6574 +", + ) + ); + Ok(()) +} From d180e0787c4166e84fb3b678b181799790737e49 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 00:05:39 +0200 Subject: [PATCH 2/4] cultmesh-rs: a put's sink may answer later; the ack waits for the answer The raw document sink gets a reply handle, CultMeshRudpPutReply, with accept() and refuse(reason). It can be answered now or later, from any thread; dropping it unanswered refuses the put ("the put was not answered"). A closure sink answers at once, and its server sends the same datagrams as before (pinned by the byte test landed before this). A put's transport acknowledgement is its answer (Q5-A): while any put a session sent is unanswered, the server sends that session nothing new. No end-of-poll ack, no Pong, no reply to a repeated Connect, no snapshot response. Every packet built now carries ack fields covering every received sequence, so each of these would acknowledge the put. Resends of packets built earlier still go. One function, acknowledgement_withheld, decides it. Answers go through a channel and are handled at the start of each poll, and again as a session's waiting messages are served. An accepted answer that leaves the session with no unanswered put releases one ack. A refusal ends the session with put-serve's fixed refusal and goodbye, acknowledging nothing, and is returned as that poll's rejection. Acks are cumulative, so a later put's answer never leaves before an earlier one's. A session's delivered messages are handled in order. A put is offered to the sink at once, even behind a pending put, so pipelined puts can share one write. A snapshot request waits for every earlier put's answer. A put that arrives behind such a request is unanswered too, so the request's response is built when its turn comes and held until that put is answered. Held puts are bounded by the existing session and byte limits (Q5-B). Every session removal goes through remove_session, which forgets the session's pending puts, so their answers are discarded and cannot reach a later generation of the same address and connection id. Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-rs/src/rudp_document_server.rs | 504 ++++++++++++--- .../cultmesh-rs/tests/rudp_document_server.rs | 585 +++++++++++++++++- 2 files changed, 970 insertions(+), 119 deletions(-) diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 09c31adf..3b79e030 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -5,11 +5,10 @@ use cultnet_rs::{ decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, encode_rudp_packet, is_permanent_send_error, send_error_code, }; -use std::collections::BTreeMap; -#[cfg(test)] -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::io::ErrorKind; use std::net::{SocketAddr, UdpSocket}; +use std::sync::mpsc::{Receiver, Sender, channel}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const MAX_UDP_DATAGRAM_BYTES: usize = 65_535; @@ -55,16 +54,85 @@ pub struct CultMeshRudpSnapshotQuery { } /// Caller-owned admission/persistence port for received raw documents. +/// +/// The sink answers each put through its `reply`, at once or later, and from +/// any thread. The server acknowledges a put to its publisher only once the +/// sink has accepted it: "acknowledged" means the sink accepted. Until every +/// put a session sent is answered, the server sends that session nothing new, +/// only resends of what it sent before. A refused put ends its session with a +/// refusal that acknowledges nothing. +/// +/// A closure returning `Result<()>` is a sink that answers at once. pub trait CultMeshRudpRawDocumentSink { - fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()>; + fn accept_raw_document( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + reply: CultMeshRudpPutReply, + ); } impl CultMeshRudpRawDocumentSink for F where F: FnMut(CultMeshRudpRawDocumentReceipt) -> Result<()>, { - fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { - self(receipt) + fn accept_raw_document( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + reply: CultMeshRudpPutReply, + ) { + match self(receipt) { + Ok(()) => reply.accept(), + Err(error) => reply.refuse(format!("{error:#}")), + } + } +} + +/// The reason a put is refused when its reply is dropped unanswered. +pub const UNANSWERED_PUT_REASON: &str = "the put was not answered"; + +/// A sink's answer to one put, on its way to the server's next poll. +struct PutAnswer { + put: u64, + refusal: Option, +} + +/// The answer a sink owes one put. Answer it with `accept` or `refuse`, now or +/// later, from any thread; the server tells the publisher on its next poll. +/// Dropping it unanswered refuses the put (`UNANSWERED_PUT_REASON`). An answer +/// that reaches a server whose session for the put has ended is discarded. +#[derive(Debug)] +#[must_use = "a put whose reply is dropped unanswered is refused"] +pub struct CultMeshRudpPutReply { + put: u64, + answers: Option>, +} + +impl CultMeshRudpPutReply { + /// The put is admitted: its publisher is acknowledged. + pub fn accept(mut self) { + self.answer(None); + } + + /// The put is refused. `reason` goes to the caller of `poll_once` as + /// `SinkRefused`; the publisher is sent only fixed text. + pub fn refuse(mut self, reason: impl std::fmt::Display) { + self.answer(Some(reason.to_string())); + } + + fn answer(&mut self, refusal: Option) { + if let Some(answers) = self.answers.take() { + // A server that is gone has nobody left to tell. + let _ = answers.send(PutAnswer { + put: self.put, + refusal, + }); + } + } +} + +impl Drop for CultMeshRudpPutReply { + fn drop(&mut self) { + self.answer(Some(UNANSWERED_PUT_REASON.into())); } } @@ -346,6 +414,35 @@ struct SessionEntry { created_at_monotonic_millis: u64, last_activity_monotonic_millis: u64, admitted_payload_bytes: usize, + /// Puts offered to the sink and not yet answered. + pending_puts: BTreeSet, + /// Delivered messages not yet handled, in the order they arrived: a + /// snapshot request waits for every earlier put's answer, and everything + /// behind it waits for it. + waiting: VecDeque<(u32, CultNetMessage)>, + /// Snapshot responses built, charged and not yet sent, because a put that + /// arrived after their request is unanswered: message id and payload. + responses_owed: VecDeque<(String, Vec)>, +} + +impl SessionEntry { + fn new(session: CultNetRudpSession, now: u64) -> Self { + Self { + session, + created_at_monotonic_millis: now, + last_activity_monotonic_millis: now, + admitted_payload_bytes: 0, + pending_puts: BTreeSet::new(), + waiting: VecDeque::new(), + responses_owed: VecDeque::new(), + } + } +} + +/// A put offered to the sink, awaiting its answer. +struct PendingPut { + session: CultMeshRudpSessionKey, + message_id: String, } /// A synchronous, multi-session CultNet RUDP raw-document server. @@ -361,6 +458,14 @@ pub struct CultMeshRudpDocumentServer { options: CultMeshRudpDocumentServerOptions, packets_dropped: u64, send_failures: u64, + /// Where replies send their answers, and the sender each reply clones. + answers: Receiver, + answer_sender: Sender, + /// Answers taken off the channel and not yet handled, in arrival order. + answered: VecDeque, + /// The next put id; ids are never reused in the server's life. + next_put: u64, + pending: BTreeMap, /// Peers whose every datagram fails to send, standing in for an unroutable /// or full path that a loopback peer cannot be made to have. #[cfg(test)] @@ -388,6 +493,7 @@ where anyhow!("CultMesh RUDP server requires a bound UDP socket: {error}") })?; socket.set_nonblocking(true)?; + let (answer_sender, answers) = channel(); Ok(Self { socket, sessions: BTreeMap::new(), @@ -397,6 +503,11 @@ where options, packets_dropped: 0, send_failures: 0, + answers, + answer_sender, + answered: VecDeque::new(), + next_put: 0, + pending: BTreeMap::new(), #[cfg(test)] failing_peers: BTreeSet::new(), #[cfg(test)] @@ -441,7 +552,13 @@ where /// message. The rejection's own detail, which can quote a sink's or /// source's error, stays with the caller. Local socket and server-state /// failures remain errors. + /// + /// Sinks' answers are handled first: an accepted put is acknowledged, and + /// a refused one is returned as the poll's rejection, one per poll. pub fn poll_once(&mut self) -> Result { + if let Some(rejection) = self.serve_answered()? { + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + } self.maintain()?; let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES]; let (received, remote_addr) = match self.socket.recv_from(&mut wire) { @@ -519,49 +636,44 @@ where } }; + // The only reply `receive` returns is a Pong. Built now, it would + // acknowledge a put still awaiting its answer. let mut unsendable = match result.reply { - Some(reply) => self.send_packet(key.remote_addr, &reply)?, - None => None, + Some(reply) if !self.acknowledgement_withheld(key) => { + self.send_packet(key.remote_addr, &reply)? + } + _ => None, }; if result.disconnected { - self.sessions.remove(&key); + self.remove_session(key); return Ok(CultMeshRudpPollOutcome::Handled); } + // A reliable packet is acknowledged below once nothing is withheld. An + // unreliable one is not, so if this poll releases a session that was + // withheld, the release acknowledges it. + let ack_release = !packet.reliable && self.acknowledgement_withheld(key); + let entry = self + .sessions + .get_mut(&key) + .expect("checked session must remain present"); for frame in result.delivered { if frame.channel_id != "schema" { continue; } - let message = match decode_cultnet_message_from_slice( + if let Ok(message) = decode_cultnet_message_from_slice( &frame.payload, CultNetWireContract::CultNetSchemaV0, ) { - Ok(message) => message, - Err(_) => continue, - }; - if let Some(rejection) = self.deliver_application_message( - key, - frame.sequence, - message, - now_unix, - now_monotonic, - )? { - // The only reply `receive` returns is a Pong, which delivers no - // frame, so no earlier send of this poll can precede a rejection. - // A response that could not be sent ended its session where the - // send failed, over the real error. - if !matches!( - rejection.reason, - CultMeshRudpRejectionReason::ResponseSendFailed(_) - ) { - self.end_session(key, SessionEnd::Rejected(rejection.reason.refusal_text()))?; - } - return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + entry.waiting.push_back((frame.sequence, message)); } } + if let Some(rejection) = self.serve_session(key, now_unix, now_monotonic, ack_release)? { + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); + } - if packet.reliable && unsendable.is_none() { + if packet.reliable && unsendable.is_none() && !self.acknowledgement_withheld(key) { let ack = self .sessions .get_mut(&key) @@ -581,12 +693,18 @@ where let now = self.clock.now_monotonic_millis(); let idle_timeout_ms = duration_millis(self.options.session_idle_timeout); let lifetime_ms = duration_millis(self.options.session_max_lifetime); - let before = self.sessions.len(); - self.sessions.retain(|_, entry| { - now.saturating_sub(entry.last_activity_monotonic_millis) <= idle_timeout_ms - && now.saturating_sub(entry.created_at_monotonic_millis) <= lifetime_ms - }); - let expired = before - self.sessions.len(); + let expired: Vec<_> = self + .sessions + .iter() + .filter(|(_, entry)| { + now.saturating_sub(entry.last_activity_monotonic_millis) > idle_timeout_ms + || now.saturating_sub(entry.created_at_monotonic_millis) > lifetime_ms + }) + .map(|(key, _)| *key) + .collect(); + for key in &expired { + self.remove_session(*key); + } let mut resends = Vec::new(); for (key, entry) in &mut self.sessions { @@ -604,7 +722,7 @@ where } } Ok(CultMeshRudpMaintenance { - sessions_expired: expired, + sessions_expired: expired.len(), packets_resent: resends.len(), }) } @@ -625,7 +743,7 @@ where .get(&key) .is_some_and(|entry| !entry.session.connect_repeats(packet)) { - self.sessions.remove(&key); + self.remove_session(key); } let reply = match self.sessions.get_mut(&key) { Some(entry) => { @@ -651,18 +769,15 @@ where ), }); let accept = session.accept_connect(packet, now, Vec::new())?; - self.sessions.insert( - key, - SessionEntry { - session, - created_at_monotonic_millis: now, - last_activity_monotonic_millis: now, - admitted_payload_bytes: 0, - }, - ); + self.sessions.insert(key, SessionEntry::new(session, now)); accept } }; + // A repeated Connect is answered with an acknowledgement built now, + // which would acknowledge a put still awaiting its answer. + if self.acknowledgement_withheld(key) { + return Ok(true); + } if let Some(error) = self.send_packet(key.remote_addr, &reply)? { self.end_session(key, SessionEnd::Unsendable(error))?; return Ok(false); @@ -699,9 +814,9 @@ where let served = match self.snapshot_source.served_record(&document) { Ok(served) => served, Err(error) => { - return reject(CultMeshRudpRejectionReason::SnapshotSourceFailed( - format!("{error:#}"), - )); + return reject(CultMeshRudpRejectionReason::SnapshotSourceFailed(format!( + "{error:#}" + ))); } }; let alone = CultNetMessage::SnapshotResponseRaw { @@ -726,18 +841,32 @@ where max_fragment_count, }); } + let put = self.next_put; + self.next_put += 1; + self.sessions + .get_mut(&key) + .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before a put"))? + .pending_puts + .insert(put); + self.pending.insert( + put, + PendingPut { + session: key, + message_id: message_id.clone(), + }, + ); let receipt = CultMeshRudpRawDocumentReceipt { session: key, - message_id: message_id.clone(), + message_id, transport_sequence, received_at_unix_millis: now_unix, document, }; - if let Err(error) = self.sink.accept_raw_document(receipt) { - return reject(CultMeshRudpRejectionReason::SinkRefused(format!( - "{error:#}" - ))); - } + let reply = CultMeshRudpPutReply { + put, + answers: Some(self.answer_sender.clone()), + }; + self.sink.accept_raw_document(receipt, reply); } CultNetMessage::SnapshotRequest { message_id, @@ -809,55 +938,237 @@ where reason: CultMeshRudpRejectionReason::PayloadBudgetFull, })); } - let payload_bytes = payload.len(); + let withheld = self.acknowledgement_withheld(key); let entry = self .sessions .get_mut(&key) .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before response"))?; - let packets = match entry.session.send_many( - "schema", - payload, - CultNetRudpSendOptions { - reliable: true, - ordered: true, - sequenced: false, - now_ms: now, - reliable_expire_after_ms: None, - }, - Some(self.options.max_fragment_bytes), - ) { - Ok(packets) => packets, - Err(error) => { - return Ok(Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::SnapshotRequest, - message_id, - reason: CultMeshRudpRejectionReason::ResponseQueueFailed(format!( - "{error:#}" - )), - })); - } - }; entry.admitted_payload_bytes = - entry.admitted_payload_bytes.saturating_add(payload_bytes); - for packet in &packets { - if let Some(error) = self.send_packet(key.remote_addr, packet)? { - let code = send_error_code(&error); - self.end_session(key, SessionEnd::Unsendable(error))?; - return Ok(Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::SnapshotRequest, - message_id, - reason: CultMeshRudpRejectionReason::ResponseSendFailed(code), - })); - } + entry.admitted_payload_bytes.saturating_add(payload.len()); + if withheld { + entry.responses_owed.push_back((message_id, payload)); + return Ok(None); } + return self.send_response(key, message_id, payload, now); } _ => {} } Ok(None) } + /// Queues a charged snapshot response on its session and sends it. + fn send_response( + &mut self, + key: CultMeshRudpSessionKey, + message_id: String, + payload: Vec, + now: u64, + ) -> Result> { + let rejection = |message_id, reason| { + Ok(Some(CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::SnapshotRequest, + message_id, + reason, + })) + }; + let entry = self + .sessions + .get_mut(&key) + .ok_or_else(|| anyhow!("CultMesh RUDP session disappeared before response"))?; + let packets = match entry.session.send_many( + "schema", + payload, + CultNetRudpSendOptions { + reliable: true, + ordered: true, + sequenced: false, + now_ms: now, + reliable_expire_after_ms: None, + }, + Some(self.options.max_fragment_bytes), + ) { + Ok(packets) => packets, + Err(error) => { + return rejection( + message_id, + CultMeshRudpRejectionReason::ResponseQueueFailed(format!("{error:#}")), + ); + } + }; + for packet in &packets { + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + let code = send_error_code(&error); + self.end_session(key, SessionEnd::Unsendable(error))?; + return rejection( + message_id, + CultMeshRudpRejectionReason::ResponseSendFailed(code), + ); + } + } + Ok(None) + } + + /// Whether the session may be sent nothing new: a put it sent awaits its + /// answer, offered to the sink or waiting behind a snapshot request. Every + /// packet built now carries acknowledgement fields covering every sequence + /// received, so it would acknowledge that put. Resends of packets built + /// earlier carry their own fields and still go. The one owner of the rule. + fn acknowledgement_withheld(&self, key: CultMeshRudpSessionKey) -> bool { + self.sessions.get(&key).is_some_and(|entry| { + !entry.pending_puts.is_empty() + || entry + .waiting + .iter() + .any(|(_, message)| matches!(message, CultNetMessage::DocumentPutRaw { .. })) + }) + } + + /// Handles the sinks' answers that arrived since the last poll, session by + /// session, and returns the first refusal. + fn serve_answered(&mut self) -> Result> { + self.answered.extend(self.answers.try_iter()); + let now_unix = self.clock.now_unix_millis(); + let now = self.clock.now_monotonic_millis(); + while let Some(key) = self + .answered + .iter() + .find_map(|answer| self.pending.get(&answer.put).map(|put| put.session)) + { + // A session with an answer queued has a pending put: it is withheld. + if let Some(rejection) = self.serve_session(key, now_unix, now, true)? { + return Ok(Some(rejection)); + } + } + // What is left answers puts whose sessions ended. + self.answered.clear(); + Ok(None) + } + + /// Takes the session's answers and handles its waiting messages in order, + /// until it waits on the sink or has nothing left. A put is offered at + /// once, even behind a put awaiting its answer; a snapshot request waits + /// for every earlier put's answer. When the session stops being withheld, + /// the responses owed to it are sent, after an acknowledgement if + /// `ack_owed`. + fn serve_session( + &mut self, + key: CultMeshRudpSessionKey, + now_unix: u64, + now: u64, + mut ack_owed: bool, + ) -> Result> { + loop { + if let Some(rejection) = self.take_answers(key)? { + return Ok(Some(rejection)); + } + if !self.acknowledgement_withheld(key) { + if std::mem::take(&mut ack_owed) { + let Some(entry) = self.sessions.get_mut(&key) else { + return Ok(None); + }; + let ack = entry.session.create_ack(); + if let Some(error) = self.send_packet(key.remote_addr, &ack)? { + self.end_session(key, SessionEnd::Unsendable(error))?; + return Ok(None); + } + } + while let Some((message_id, payload)) = self + .sessions + .get_mut(&key) + .and_then(|entry| entry.responses_owed.pop_front()) + { + if let Some(rejection) = self.send_response(key, message_id, payload, now)? { + return self.reject(key, rejection); + } + } + } + let Some(entry) = self.sessions.get_mut(&key) else { + return Ok(None); + }; + let ready = match entry.waiting.front() { + None => false, + Some((_, CultNetMessage::SnapshotRequest { .. })) => entry.pending_puts.is_empty(), + Some(_) => true, + }; + let Some((sequence, message)) = ready.then(|| entry.waiting.pop_front()).flatten() + else { + return Ok(None); + }; + if let Some(rejection) = + self.deliver_application_message(key, sequence, message, now_unix, now)? + { + return self.reject(key, rejection); + } + } + } + + /// Takes the session's answers off the queue. An accepted put stops being + /// pending; a refused one ends the session and is returned. Answers for + /// puts whose session ended are discarded; other sessions' stay queued. + fn take_answers( + &mut self, + key: CultMeshRudpSessionKey, + ) -> Result> { + self.answered.extend(self.answers.try_iter()); + let mut index = 0; + while index < self.answered.len() { + let put = self.answered[index].put; + match self.pending.get(&put) { + Some(pending) if pending.session != key => { + index += 1; + continue; + } + Some(_) => {} + None => { + self.answered.remove(index); + continue; + } + } + let answer = self.answered.remove(index).expect("indexed answer"); + let pending = self.pending.remove(&put).expect("pending put"); + if let Some(entry) = self.sessions.get_mut(&key) { + entry.pending_puts.remove(&put); + } + if let Some(reason) = answer.refusal { + let rejection = CultMeshRudpApplicationRejection { + session: key, + operation: CultMeshRudpApplicationOperation::DocumentPutRaw, + message_id: pending.message_id, + reason: CultMeshRudpRejectionReason::SinkRefused(reason), + }; + return self.reject(key, rejection); + } + } + Ok(None) + } + + /// Ends the session over `rejection` and returns it. A response that could + /// not be sent ended its session where the send failed, over the real error. + fn reject( + &mut self, + key: CultMeshRudpSessionKey, + rejection: CultMeshRudpApplicationRejection, + ) -> Result> { + if !matches!( + rejection.reason, + CultMeshRudpRejectionReason::ResponseSendFailed(_) + ) { + self.end_session(key, SessionEnd::Rejected(rejection.reason.refusal_text()))?; + } + Ok(Some(rejection)) + } + + /// Removes a session and forgets its pending puts, so their answers are + /// discarded. Every removal goes through here. + fn remove_session(&mut self, key: CultMeshRudpSessionKey) -> Option { + let entry = self.sessions.remove(&key)?; + for put in &entry.pending_puts { + self.pending.remove(put); + } + Some(entry) + } + /// Ends a session before its peer is done with it, and says goodbye. It is /// the one way a session ends early: every end removes the session, sends /// what the peer is owed, then the goodbye. @@ -870,7 +1181,7 @@ where /// that can never be sent as built is not dropped: the goodbye names its /// error instead. A refusal that cannot be encoded is not sent. fn end_session(&mut self, key: CultMeshRudpSessionKey, end: SessionEnd) -> Result<()> { - let Some(mut entry) = self.sessions.remove(&key) else { + let Some(mut entry) = self.remove_session(key) else { return Ok(()); }; let (refusal, mut unsendable) = match end { @@ -1069,7 +1380,6 @@ fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { Ok(()) } - /// The one encoder for a snapshot response, shared by the snapshot path and by /// put admission so the size a put is judged by is the size it would be served at. fn encode_snapshot_response(response: &CultNetMessage) -> std::result::Result, String> { diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index c9aaebc4..63a9080f 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -1,9 +1,10 @@ use anyhow::Result; use cultmesh_rs::{ CultMeshRudpApplicationOperation, CultMeshRudpDocumentServer, - CultMeshRudpDocumentServerOptions, CultMeshRudpPollOutcome, CultMeshRudpRawDocumentReceipt, - CultMeshRudpRawDocumentSink, CultMeshRudpRejectionReason, CultMeshRudpServerClock, - CultMeshRudpSnapshotQuery, CultMeshRudpSnapshotSource, + CultMeshRudpDocumentServerOptions, CultMeshRudpPollOutcome, CultMeshRudpPutReply, + CultMeshRudpRawDocumentReceipt, CultMeshRudpRawDocumentSink, CultMeshRudpRejectionReason, + CultMeshRudpServerClock, CultMeshRudpSnapshotQuery, CultMeshRudpSnapshotSource, + UNANSWERED_PUT_REASON, }; use cultnet_rs::{ CultNetMessage, CultNetRawDocumentRecord, CultNetRawPayloadEncoding, @@ -37,14 +38,19 @@ impl Sink { } impl CultMeshRudpRawDocumentSink for Sink { - fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { + fn accept_raw_document( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + reply: CultMeshRudpPutReply, + ) { let mut state = self.0.lock().unwrap(); if state.failures_remaining > 0 { state.failures_remaining -= 1; - anyhow::bail!("injected sink failure CANARY-7f3a"); + reply.refuse("injected sink failure CANARY-7f3a"); + return; } state.receipts.push(receipt); - Ok(()) + reply.accept(); } } @@ -115,7 +121,12 @@ impl CultMeshRudpSnapshotSource for StrippingSource { &mut self, query: &CultMeshRudpSnapshotQuery, ) -> Result> { - Ok(self.inner.raw_snapshot(query)?.iter().map(stripped).collect()) + Ok(self + .inner + .raw_snapshot(query)? + .iter() + .map(stripped) + .collect()) } fn served_record( @@ -241,8 +252,8 @@ fn client_on( }) } -fn connect( - server: &mut CultMeshRudpDocumentServer, +fn connect( + server: &mut CultMeshRudpDocumentServer, clients: &mut [&mut CultNetRudpSocketTransportConnection], ) -> Result<()> { for client in &mut *clients { @@ -1240,7 +1251,13 @@ fn options_that_cannot_carry_an_empty_snapshot_response_are_refused() -> Result< )? .len(); let build = |options: CultMeshRudpDocumentServerOptions| { - server(options, Clock::new(63_000), Sink::default(), Source::default()).map(|_| ()) + server( + options, + Clock::new(63_000), + Sink::default(), + Source::default(), + ) + .map(|_| ()) }; let bytes = |max_snapshot_response_bytes| CultMeshRudpDocumentServerOptions { max_snapshot_response_bytes, @@ -1397,6 +1414,8 @@ struct RawPeer { socket: UdpSocket, target: SocketAddr, session: cultnet_rs::CultNetRudpSession, + /// The Connect that started the session, for a test to repeat. + connect: cultnet_rs::CultNetRudpPacket, } impl RawPeer { @@ -1406,26 +1425,58 @@ impl RawPeer { ) -> Result { let socket = UdpSocket::bind("127.0.0.1:0")?; socket.set_read_timeout(Some(Duration::from_millis(50)))?; - let mut peer = Self { + let target = server.local_addr()?; + let (session, connect) = Self::handshake(&socket, target, server, connection_id, 100)?; + Ok(Self { socket, - target: server.local_addr()?, - session: cultnet_rs::CultNetRudpSession::new(cultnet_rs::CultNetRudpSessionOptions { + target, + session, + connect, + }) + } + + /// Connects again from the same address and connection id, starting a + /// new generation of the session. + fn reconnect( + &mut self, + server: &mut CultMeshRudpDocumentServer, + ) -> Result<()> { + let connection_id = self.session.connection_id(); + (self.session, self.connect) = + Self::handshake(&self.socket, self.target, server, connection_id, 5_000)?; + Ok(()) + } + + fn handshake( + socket: &UdpSocket, + target: SocketAddr, + server: &mut CultMeshRudpDocumentServer, + connection_id: u32, + initial_sequence: u32, + ) -> Result<( + cultnet_rs::CultNetRudpSession, + cultnet_rs::CultNetRudpPacket, + )> { + let mut session = + cultnet_rs::CultNetRudpSession::new(cultnet_rs::CultNetRudpSessionOptions { connection_id, - initial_sequence: Some(100), + initial_sequence: Some(initial_sequence), resend_delay_ms: 10_000, max_pending_reliable_packets: Some(64), - }), - }; - let connect = peer.session.create_connect(0, Vec::new())?; - peer.send_packet(&connect)?; + }); + let connect = session.create_connect(0, Vec::new())?; + socket.send_to(&cultnet_rs::encode_rudp_packet(&connect)?, target)?; server.poll_once()?; - let accept = peer.receive().expect("the server accepts the connect"); + let mut wire = vec![0_u8; 65_535]; + let (bytes, _) = socket.recv_from(&mut wire)?; + let accept = cultnet_rs::decode_rudp_packet(&wire[..bytes])?; assert_eq!( accept.packet_type, - cultnet_rs::CultNetRudpPacketType::Accept + cultnet_rs::CultNetRudpPacketType::Accept, + "the server accepts the connect" ); - peer.session.receive(&accept, 1)?; - Ok(peer) + session.receive(&accept, 1)?; + Ok((session, connect)) } fn send_packet(&self, packet: &cultnet_rs::CultNetRudpPacket) -> Result<()> { @@ -1831,3 +1882,493 @@ fn a_closure_sink_sends_the_same_datagrams_as_before_replies_could_wait() -> Res ); Ok(()) } + +/// A sink that holds every put's reply for the test to answer. +#[derive(Clone, Default)] +struct HeldSink(Arc>); + +#[derive(Default)] +struct HeldState { + offered: Vec, + replies: Vec<(String, CultMeshRudpPutReply)>, +} + +impl CultMeshRudpRawDocumentSink for HeldSink { + fn accept_raw_document( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + reply: CultMeshRudpPutReply, + ) { + let mut state = self.0.lock().unwrap(); + state.offered.push(receipt.message_id.clone()); + state.replies.push((receipt.message_id, reply)); + } +} + +impl HeldSink { + /// The message ids of the puts offered so far, in order. + fn offered(&self) -> Vec { + self.0.lock().unwrap().offered.clone() + } + + /// The held reply for the put `message_id`. + fn reply(&self, message_id: &str) -> CultMeshRudpPutReply { + let mut state = self.0.lock().unwrap(); + let index = state + .replies + .iter() + .position(|(id, _)| id == message_id) + .expect("the put was offered and is unanswered"); + state.replies.remove(index).1 + } +} + +type HeldServer = CultMeshRudpDocumentServer; + +fn held_server(sink: &HeldSink, clock: &Clock) -> Result { + CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + sink.clone(), + Source::documents(vec![document("held", vec![1, 2, 3])]), + clock.clone(), + Default::default(), + ) +} + +fn put(message_id: &str) -> CultNetMessage { + CultNetMessage::DocumentPutRaw { + message_id: message_id.into(), + document: document(message_id, vec![4, 5, 6]), + } +} + +fn snapshot_request(message_id: &str) -> CultNetMessage { + CultNetMessage::SnapshotRequest { + message_id: message_id.into(), + schema_ids: None, + record_keys: None, + } +} + +/// Polls until the server has nothing left to read. +fn poll_until_idle( + server: &mut CultMeshRudpDocumentServer, +) -> Result<()> { + for _ in 0..100 { + match server.poll_once()? { + CultMeshRudpPollOutcome::Idle => return Ok(()), + CultMeshRudpPollOutcome::Handled => {} + rejected => panic!("polling must not reject: {rejected:?}"), + } + } + anyhow::bail!("the server never went idle") +} + +/// The CultNet message a single-packet schema frame carries. +fn schema_message(packet: &cultnet_rs::CultNetRudpPacket) -> Result { + assert_eq!(packet.fragment_count, 0, "fixture: one packet per message"); + decode_cultnet_message_from_slice(&packet.payload, CultNetWireContract::CultNetSchemaV0) +} + +fn assert_acknowledges_none(packets: &[cultnet_rs::CultNetRudpPacket], sequences: &[u32]) { + for packet in packets { + assert!( + sequences + .iter() + .all(|sequence| !acknowledges(packet, *sequence)), + "{:?} acknowledges an unanswered put: ack {} mask {:#x}", + packet.packet_type, + packet.ack, + packet.ack_mask + ); + } +} + +/// While a put awaits its answer, its session is sent nothing new: no +/// acknowledgement, no Pong, no snapshot response, across five resend +/// intervals. A response built before the put still resends, carrying the +/// acknowledgement fields it was built with. Once the put is accepted, the +/// next poll acknowledges it, then answers the snapshot request that waited. +#[test] +fn a_held_put_is_acknowledged_only_after_it_is_accepted() -> Result<()> { + let clock = Clock::new(80_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 90)?; + + // A response the peer never acknowledges, built before the put. + peer.send(&snapshot_request("before"))?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + let earlier: Vec = peer + .drain() + .iter() + .filter(|packet| packet.packet_type == cultnet_rs::CultNetRudpPacketType::Data) + .map(|packet| packet.sequence) + .collect(); + assert_eq!( + earlier.len(), + 1, + "fixture: the earlier response is one packet" + ); + + let held = peer.send(&put("held"))?; + poll_until_idle(&mut server)?; + assert_eq!(sink.offered(), vec!["held".to_string()]); + let mut seen = peer.drain(); + for step in 1..=5 { + clock.set(80_000 + 50 * step); + let ping = peer.session.create_ping(b"alive".to_vec()); + peer.send_packet(&ping)?; + if step == 3 { + peer.send(&snapshot_request("after"))?; + } + poll_until_idle(&mut server)?; + seen.extend(peer.drain()); + } + assert_acknowledges_none(&seen, &held); + let kinds: Vec<_> = seen + .iter() + .map(|packet| (packet.packet_type, packet.sequence)) + .collect(); + assert_eq!( + kinds, + vec![(cultnet_rs::CultNetRudpPacketType::Data, earlier[0]); 5], + "only the earlier response's resends reach the peer" + ); + + sink.reply("held").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + let released = peer.drain(); + assert_eq!(released.len(), 2, "{released:?}"); + assert_eq!( + released[0].packet_type, + cultnet_rs::CultNetRudpPacketType::Ack + ); + assert!( + held.iter() + .all(|sequence| acknowledges(&released[0], *sequence)) + ); + let CultNetMessage::SnapshotResponseRaw { message_id, .. } = schema_message(&released[1])? + else { + panic!( + "the waiting snapshot request is answered: {:?}", + released[1] + ); + }; + assert_eq!(message_id, "after"); + Ok(()) +} + +/// A put awaiting its answer holds only its own session: another session's +/// snapshot request is answered in the same polls. +#[test] +fn another_session_is_served_while_a_put_is_held() -> Result<()> { + let clock = Clock::new(81_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut holding = RawPeer::connect(&mut server, 91)?; + let mut reading = RawPeer::connect(&mut server, 92)?; + let held = holding.send(&put("held"))?; + poll_until_idle(&mut server)?; + let read = reading.send(&snapshot_request("read"))?; + poll_until_idle(&mut server)?; + + let answered = reading.drain(); + let response = answered + .iter() + .find(|packet| packet.packet_type == cultnet_rs::CultNetRudpPacketType::Data) + .expect("the other session is answered"); + assert!(matches!( + schema_message(response)?, + CultNetMessage::SnapshotResponseRaw { .. } + )); + assert!(acknowledges(response, read[0])); + assert_acknowledges_none(&holding.drain(), &held); + Ok(()) +} + +/// A refused reply refuses the put: the poll returns the sink's reason, and +/// the publisher gets the fixed refusal and a goodbye, and never an +/// acknowledgement. +#[test] +fn a_refused_reply_refuses_the_put_and_acknowledges_nothing() -> Result<()> { + let clock = Clock::new(82_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut publisher = client(server.local_addr()?, 93)?; + connect(&mut server, &mut [&mut publisher])?; + let receipt = send_reliable(&mut publisher, &put("refused"))?; + for _ in 0..200 { + publisher.receive_once()?; + publisher.poll_resends()?; + server.poll_once()?; + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(sink.offered(), vec!["refused".to_string()]); + assert_eq!( + publisher.reliable_send_status(&receipt), + CultNetRudpReliableSendStatus::Pending, + "a held put is not acknowledged, however often it is resent" + ); + + sink.reply("refused") + .refuse("the store said no CANARY-7f3a"); + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("the refusal is the next poll's outcome"); + }; + assert_eq!( + (rejection.operation, rejection.message_id, rejection.reason), + ( + CultMeshRudpApplicationOperation::DocumentPutRaw, + "refused".to_string(), + CultMeshRudpRejectionReason::SinkRefused("the store said no CANARY-7f3a".into()) + ) + ); + assert_eq!( + refusal_seen_by(&mut publisher, &receipt)?, + "the catalog refused the document" + ); + assert_eq!(server.session_count(), 0); + Ok(()) +} + +/// A reply dropped unanswered refuses its put, over a reason of its own. +#[test] +fn a_dropped_reply_refuses_the_put() -> Result<()> { + let clock = Clock::new(83_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 94)?; + let held = peer.send(&put("dropped"))?; + poll_until_idle(&mut server)?; + + drop(sink.reply("dropped")); + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("a dropped reply refuses the put"); + }; + assert_eq!( + rejection.reason, + CultMeshRudpRejectionReason::SinkRefused(UNANSWERED_PUT_REASON.into()) + ); + assert_eq!(UNANSWERED_PUT_REASON, "the put was not answered"); + let sent = peer.drain(); + assert_acknowledges_none(&sent, &held); + let kinds: Vec<_> = sent.iter().map(|packet| packet.packet_type).collect(); + assert_eq!( + kinds, + vec![ + cultnet_rs::CultNetRudpPacketType::Data, + cultnet_rs::CultNetRudpPacketType::Disconnect + ] + ); + assert_eq!( + schema_message(&sent[0])?, + CultNetMessage::Error { + error: "the catalog refused the document".into(), + code: None, + details: None, + } + ); + Ok(()) +} + +/// Two pipelined puts are both offered at once, so one write can cover them. +/// Answering the later first releases nothing; answering the earlier then +/// releases one acknowledgement covering both. A later put's acknowledgement +/// never leaves before an earlier one's. +#[test] +fn a_later_puts_answer_never_leaves_before_an_earlier_ones() -> Result<()> { + let clock = Clock::new(84_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 95)?; + let first = peer.send(&put("first"))?; + let second = peer.send(&put("second"))?; + poll_until_idle(&mut server)?; + assert_eq!(sink.offered(), vec!["first".to_string(), "second".into()]); + + sink.reply("second").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!( + peer.drain(), + Vec::new(), + "the later answer releases nothing" + ); + + sink.reply("first").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + let released = peer.drain(); + assert_eq!(released.len(), 1, "{released:?}"); + assert_eq!( + released[0].packet_type, + cultnet_rs::CultNetRudpPacketType::Ack + ); + for sequence in first.iter().chain(&second) { + assert!(acknowledges(&released[0], *sequence)); + } + Ok(()) +} + +/// A put behind a snapshot request that waits on an earlier put is not +/// offered until that request is answered, and the answer is not sent +/// while the put is unanswered: it would acknowledge the put. +#[test] +fn a_put_behind_a_waiting_snapshot_request_is_acknowledged_only_once_accepted() -> Result<()> { + let clock = Clock::new(85_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 96)?; + let first = peer.send(&put("first"))?; + peer.send(&snapshot_request("between"))?; + let second = peer.send(&put("second"))?; + poll_until_idle(&mut server)?; + assert_eq!( + sink.offered(), + vec!["first".to_string()], + "the second put waits behind the snapshot request" + ); + + sink.reply("first").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!(sink.offered(), vec!["first".to_string(), "second".into()]); + assert_eq!( + peer.drain(), + Vec::new(), + "nothing is sent while the second put is unanswered" + ); + + sink.reply("second").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + let released = peer.drain(); + assert_eq!(released.len(), 2, "{released:?}"); + assert_eq!( + released[0].packet_type, + cultnet_rs::CultNetRudpPacketType::Ack + ); + for sequence in first.iter().chain(&second) { + assert!(acknowledges(&released[0], *sequence)); + } + let CultNetMessage::SnapshotResponseRaw { message_id, .. } = schema_message(&released[1])? + else { + panic!("the snapshot request is answered: {:?}", released[1]); + }; + assert_eq!(message_id, "between"); + Ok(()) +} + +/// A repeated Connect is answered with an acknowledgement built now, so while +/// a put awaits its answer the repeat is answered with nothing. +#[test] +fn a_repeated_connect_is_not_answered_while_a_put_is_held() -> Result<()> { + let clock = Clock::new(86_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut peer = RawPeer::connect(&mut server, 97)?; + let held = peer.send(&put("held"))?; + poll_until_idle(&mut server)?; + peer.send_packet(&peer.connect)?; + poll_until_idle(&mut server)?; + assert_eq!(peer.drain(), Vec::new()); + + sink.reply("held").accept(); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + let released = peer.drain(); + assert_eq!(released.len(), 1, "{released:?}"); + assert!( + held.iter() + .all(|sequence| acknowledges(&released[0], *sequence)) + ); + Ok(()) +} + +/// When a session ends, the answers its puts still owe are discarded. A +/// refusal owed by an earlier generation of the same address and connection +/// id does not end the new one, and an acceptance owed by a session that +/// disconnected or expired sends nothing. +#[test] +fn a_session_that_ends_discards_the_answers_it_is_owed() -> Result<()> { + let clock = Clock::new(87_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + + // A new generation replaces the one holding the put. + let mut replaced = RawPeer::connect(&mut server, 98)?; + replaced.send(&put("replaced"))?; + poll_until_idle(&mut server)?; + replaced.reconnect(&mut server)?; + sink.reply("replaced").refuse("stale CANARY-7f3a"); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!(replaced.drain(), Vec::new()); + let ping = replaced.session.create_ping(b"new".to_vec()); + replaced.send_packet(&ping)?; + poll_until_idle(&mut server)?; + let answered = replaced.drain(); + assert_eq!( + answered + .iter() + .map(|packet| packet.packet_type) + .collect::>(), + vec![cultnet_rs::CultNetRudpPacketType::Pong], + "the new generation is served and is not withheld" + ); + + // The peer says goodbye while its put is held. + let mut departed = RawPeer::connect(&mut server, 99)?; + departed.send(&put("departed"))?; + poll_until_idle(&mut server)?; + let goodbye = departed.session.create_disconnect(Vec::new()); + departed.send_packet(&goodbye)?; + poll_until_idle(&mut server)?; + + // The session idles out while its put is held. + let mut expired = RawPeer::connect(&mut server, 100)?; + expired.send(&put("expired"))?; + poll_until_idle(&mut server)?; + clock.set(87_000 + 31_000); + poll_until_idle(&mut server)?; + assert_eq!(server.session_count(), 0); + + sink.reply("departed").accept(); + drop(sink.reply("expired")); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!(departed.drain(), Vec::new()); + assert_eq!(expired.drain(), Vec::new()); + Ok(()) +} + +/// A put the server could never serve is refused before the sink sees it. +#[test] +fn an_unservable_put_is_never_offered_to_the_sink() -> Result<()> { + let clock = Clock::new(88_000); + let sink = HeldSink::default(); + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + sink.clone(), + Source::default(), + clock, + CultMeshRudpDocumentServerOptions { + max_snapshot_response_bytes: 1_000, + ..Default::default() + }, + )?; + let mut peer = RawPeer::connect(&mut server, 101)?; + peer.send(&CultNetMessage::DocumentPutRaw { + message_id: "big".into(), + document: document("big", vec![7; 2_000]), + })?; + // The put arrives in two fragments. + let mut rejection = None; + for _ in 0..10 { + if let CultMeshRudpPollOutcome::ApplicationRejected(found) = server.poll_once()? { + rejection = Some(found); + break; + } + } + let rejection = rejection.expect("the put can never be served"); + assert!(matches!( + rejection.reason, + CultMeshRudpRejectionReason::DocumentUnservable { .. } + )); + assert_eq!(sink.offered(), Vec::::new()); + Ok(()) +} From 9d95e7516d230241d9b3b5ed986e17b2dcb3e4cd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 00:39:52 +0200 Subject: [PATCH 3/4] cultmesh-rs: pin that a release inside a poll sends exactly one ack A sink can answer an earlier put while it is being offered a later one, in the same poll. If that poll's packet is unreliable it has no ack of its own, so the release must send one. If it is reliable, its own end-of-poll ack is the only one. A sink that accepts every held reply when it is offered the put "last" drives both cases. Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-rs/tests/rudp_document_server.rs | 64 ++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 63a9080f..340d0abb 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -1487,10 +1487,14 @@ impl RawPeer { /// Sends `message` reliably and in order; returns its sequences. fn send(&mut self, message: &CultNetMessage) -> Result> { + self.send_with(message, true) + } + + fn send_with(&mut self, message: &CultNetMessage, reliable: bool) -> Result> { let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; let options = cultnet_rs::CultNetRudpSendOptions { - reliable: true, - ordered: true, + reliable, + ordered: reliable, sequenced: false, now_ms: 2, reliable_expire_after_ms: None, @@ -2372,3 +2376,59 @@ fn an_unservable_put_is_never_offered_to_the_sink() -> Result<()> { assert_eq!(sink.offered(), Vec::::new()); Ok(()) } + +/// A sink that holds each put's reply until it is offered the put `last`, +/// then accepts every held reply and that one, inside the same poll. +#[derive(Default)] +struct AnswersOnLast(Vec); + +impl CultMeshRudpRawDocumentSink for AnswersOnLast { + fn accept_raw_document( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + reply: CultMeshRudpPutReply, + ) { + self.0.push(reply); + if receipt.message_id == "last" { + self.0.drain(..).for_each(CultMeshRudpPutReply::accept); + } + } +} + +/// A poll whose packet releases a withheld session sends exactly one +/// acknowledgement. A reliable packet's own acknowledgement is that one; an +/// unreliable packet has none, so the release sends it. +#[test] +fn a_release_inside_a_poll_sends_exactly_one_acknowledgement() -> Result<()> { + for reliable in [false, true] { + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + AnswersOnLast::default(), + Source::default(), + Clock::new(89_000), + Default::default(), + )?; + let mut peer = RawPeer::connect(&mut server, 102)?; + let first = peer.send(&put("first"))?; + poll_until_idle(&mut server)?; + assert_eq!(peer.drain(), Vec::new(), "fixture: the first put is held"); + + let last = peer.send_with(&put("last"), reliable)?; + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); + let sent = peer.drain(); + assert_eq!(sent.len(), 1, "reliable {reliable}: {sent:?}"); + assert_eq!(sent[0].packet_type, cultnet_rs::CultNetRudpPacketType::Ack); + assert!( + first + .iter() + .all(|sequence| acknowledges(&sent[0], *sequence)) + ); + if reliable { + assert!( + last.iter() + .all(|sequence| acknowledges(&sent[0], *sequence)) + ); + } + } + Ok(()) +} From d708247e9b8bddfbfd7e689cf1d37e80e3b01707 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 01:03:41 +0200 Subject: [PATCH 4/4] cultmesh-rs: pin the in-poll release ack, per-session answers and expiry bounds The in-poll release test from the last commit sent its later put on an unreliable packet. The server never reads unreliable schema frames, so that put was never offered and the test failed. The release it meant to reach happens when an answer arrives from another thread while a poll runs. The test clock now accepts a held reply at its next wall-clock read, which models that. A Ping (unreliable) then gets exactly one release ack and no Pong. Reliable data on a channel the server does not read gets exactly its own end-of-poll ack. Two answers queued in one poll, an acceptance for one session and a refusal for another, each reach only their own session. This kills take_answers' session guard replaced by false. A session is kept at exactly its idle and lifetime limits and expires one millisecond past them. The expiry comparisons moved into this cut's diff when every removal went through remove_session. cargo-mutants --in-diff over the cut: 57 mutants, 44 caught, 8 unviable, 5 timeouts (take_answers scan mutants that never terminate), 0 missed. Co-Authored-By: Claude Opus 5.5 --- .../cultmesh-rs/tests/rudp_document_server.rs | 179 +++++++++++++----- 1 file changed, 131 insertions(+), 48 deletions(-) diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 340d0abb..9f793e75 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -158,6 +158,9 @@ fn stripping_server( struct Clock { unix: Arc, monotonic: Arc, + /// A reply accepted at the next wall-clock read: an answer that arrives + /// from another thread while a poll runs. + accept_on_read: Arc>>, } impl Clock { @@ -165,6 +168,7 @@ impl Clock { Self { unix: Arc::new(AtomicU64::new(now)), monotonic: Arc::new(AtomicU64::new(now)), + accept_on_read: Arc::default(), } } @@ -184,6 +188,9 @@ impl Clock { impl CultMeshRudpServerClock for Clock { fn now_unix_millis(&self) -> u64 { + if let Some(reply) = self.accept_on_read.lock().unwrap().take() { + reply.accept(); + } self.unix.load(Ordering::SeqCst) } @@ -1487,21 +1494,22 @@ impl RawPeer { /// Sends `message` reliably and in order; returns its sequences. fn send(&mut self, message: &CultNetMessage) -> Result> { - self.send_with(message, true) + let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; + self.send_on("schema", payload) } - fn send_with(&mut self, message: &CultNetMessage, reliable: bool) -> Result> { - let payload = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0)?; + /// Sends `payload` reliably and in order on `channel`; returns its sequences. + fn send_on(&mut self, channel: &str, payload: Vec) -> Result> { let options = cultnet_rs::CultNetRudpSendOptions { - reliable, - ordered: reliable, + reliable: true, + ordered: true, sequenced: false, now_ms: 2, reliable_expire_after_ms: None, }; let packets = self .session - .send_many("schema", payload, options, Some(1200))?; + .send_many(channel, payload, options, Some(1200))?; for packet in &packets { self.send_packet(packet)?; } @@ -2377,58 +2385,133 @@ fn an_unservable_put_is_never_offered_to_the_sink() -> Result<()> { Ok(()) } -/// A sink that holds each put's reply until it is offered the put `last`, -/// then accepts every held reply and that one, inside the same poll. -#[derive(Default)] -struct AnswersOnLast(Vec); - -impl CultMeshRudpRawDocumentSink for AnswersOnLast { - fn accept_raw_document( - &mut self, - receipt: CultMeshRudpRawDocumentReceipt, - reply: CultMeshRudpPutReply, - ) { - self.0.push(reply); - if receipt.message_id == "last" { - self.0.drain(..).for_each(CultMeshRudpPutReply::accept); - } - } -} - -/// A poll whose packet releases a withheld session sends exactly one -/// acknowledgement. A reliable packet's own acknowledgement is that one; an -/// unreliable packet has none, so the release sends it. +/// A poll whose packet finds its session released, by an answer that arrived +/// while the poll ran, sends exactly one acknowledgement. A reliable packet's +/// own acknowledgement is that one. An unreliable packet (a Ping) has none, +/// so the release sends it. #[test] fn a_release_inside_a_poll_sends_exactly_one_acknowledgement() -> Result<()> { for reliable in [false, true] { - let mut server = CultMeshRudpDocumentServer::new( - UdpSocket::bind("127.0.0.1:0")?, - AnswersOnLast::default(), - Source::default(), - Clock::new(89_000), - Default::default(), - )?; + let clock = Clock::new(89_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; let mut peer = RawPeer::connect(&mut server, 102)?; - let first = peer.send(&put("first"))?; + let held = peer.send(&put("held"))?; poll_until_idle(&mut server)?; - assert_eq!(peer.drain(), Vec::new(), "fixture: the first put is held"); + assert_eq!(peer.drain(), Vec::new(), "fixture: the put is held"); - let last = peer.send_with(&put("last"), reliable)?; + *clock.accept_on_read.lock().unwrap() = Some(sink.reply("held")); + let own = if reliable { + // Reliable data on a channel the server does not read. + peer.send_on("other", vec![1])? + } else { + let ping = peer.session.create_ping(Vec::new()); + peer.send_packet(&ping)?; + Vec::new() + }; assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Handled); let sent = peer.drain(); - assert_eq!(sent.len(), 1, "reliable {reliable}: {sent:?}"); - assert_eq!(sent[0].packet_type, cultnet_rs::CultNetRudpPacketType::Ack); - assert!( - first - .iter() - .all(|sequence| acknowledges(&sent[0], *sequence)) + let kinds: Vec<_> = sent.iter().map(|packet| packet.packet_type).collect(); + // No Pong: the session was withheld when the Ping arrived. + assert_eq!( + kinds, + vec![cultnet_rs::CultNetRudpPacketType::Ack], + "reliable {reliable}" ); - if reliable { - assert!( - last.iter() - .all(|sequence| acknowledges(&sent[0], *sequence)) - ); + for sequence in held.iter().chain(&own) { + assert!(acknowledges(&sent[0], *sequence)); } } Ok(()) } + +/// Answers are handled per session: one session's answer never releases or +/// ends another. One session's put is accepted and another's refused in the +/// same poll; each session gets only its own answer. +#[test] +fn one_sessions_answer_never_releases_or_ends_another() -> Result<()> { + let clock = Clock::new(90_000); + let sink = HeldSink::default(); + let mut server = held_server(&sink, &clock)?; + let mut accepted = RawPeer::connect(&mut server, 103)?; + let mut refused = RawPeer::connect(&mut server, 104)?; + let accepted_put = accepted.send(&put("accepted"))?; + let refused_put = refused.send(&put("refused"))?; + poll_until_idle(&mut server)?; + + sink.reply("accepted").accept(); + sink.reply("refused").refuse("no CANARY-7f3a"); + let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = server.poll_once()? else { + panic!("the refusal is the poll's outcome"); + }; + assert_eq!( + ( + rejection.session.connection_id, + rejection.message_id.as_str() + ), + (104, "refused") + ); + let to_accepted = accepted.drain(); + assert_eq!( + to_accepted + .iter() + .map(|packet| packet.packet_type) + .collect::>(), + vec![cultnet_rs::CultNetRudpPacketType::Ack] + ); + assert!( + accepted_put + .iter() + .all(|sequence| acknowledges(&to_accepted[0], *sequence)) + ); + let to_refused = refused.drain(); + assert_acknowledges_none(&to_refused, &refused_put); + assert_eq!( + to_refused + .iter() + .map(|packet| packet.packet_type) + .collect::>(), + vec![ + cultnet_rs::CultNetRudpPacketType::Data, + cultnet_rs::CultNetRudpPacketType::Disconnect + ] + ); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +/// A session is kept while its idle time and its age are at their limits, and +/// expires once either is past it. +#[test] +fn a_session_is_kept_at_its_limits_and_expires_past_them() -> Result<()> { + for (idle, lifetime) in [(1_000, 60_000), (60_000, 1_000)] { + let clock = Clock::new(91_000); + let mut server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + HeldSink::default(), + Source::default(), + clock.clone(), + CultMeshRudpDocumentServerOptions { + session_idle_timeout: Duration::from_millis(idle), + session_max_lifetime: Duration::from_millis(lifetime), + ..Default::default() + }, + )?; + RawPeer::connect(&mut server, 105)?; + clock.set(92_000); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!( + server.session_count(), + 1, + "kept at the limit ({idle}, {lifetime})" + ); + clock.set(92_001); + assert_eq!(server.poll_once()?, CultMeshRudpPollOutcome::Idle); + assert_eq!( + server.session_count(), + 0, + "expired past it ({idle}, {lifetime})" + ); + } + Ok(()) +}