diff --git a/packages/cultmesh-rs/src/rudp_document_server.rs b/packages/cultmesh-rs/src/rudp_document_server.rs index 647589e0..f97b2578 100644 --- a/packages/cultmesh-rs/src/rudp_document_server.rs +++ b/packages/cultmesh-rs/src/rudp_document_server.rs @@ -3,13 +3,14 @@ use cultnet_rs::{ CultNetErrorCode, CultNetErrorDetails, CultNetMessage, CultNetRawDocumentRecord, CultNetRudpPacket, CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession, CultNetRudpSessionOptions, CultNetWireContract, decode_cultnet_message_from_slice, decode_rudp_packet, encode_cultnet_message_to_vec, - encode_rudp_packet, is_permanent_send_error, send_error_code, + encode_rudp_packet, is_permanent_send_error, readable_message_id, message_id_digest, missing_required_write_field, send_error_code, write_content_hash, }; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; #[cfg(test)] use std::collections::BTreeSet; use std::io::ErrorKind; use std::net::{SocketAddr, UdpSocket}; +use std::ops::ControlFlow; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const MAX_UDP_DATAGRAM_BYTES: usize = 65_535; @@ -66,6 +67,16 @@ pub struct CultMeshRudpSnapshotQuery { /// Caller-owned admission/persistence port for received raw documents. pub trait CultMeshRudpRawDocumentSink { + /// Whether this put may be written, decided without writing it. A receipt-capable session asks it only + /// the first time it sees a message id; a resend is answered from what the session remembers + /// (write-receipt.md, rule 5). The default admits everything and leaves the decision to + /// `accept_raw_document`. + fn authorize_raw_document(&mut self, _receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> { + Ok(()) + } + + /// Commits the document. `Ok` means committed and visible to readers: a receipt-capable peer is told + /// `write_admitted` on the strength of it, so a sink that has only queued the write must not return `Ok`. fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()>; } @@ -178,6 +189,15 @@ pub struct CultMeshRudpDocumentServerOptions { pub resend_delay: Duration, pub max_pending_reliable_packets_per_session: usize, pub max_fragment_bytes: usize, + /// Server-wide memory budget for the remembered writes of all receipt sessions + /// (write-receipt.md, "Remembered-write limits"): at most this many bytes, counted as + /// `REMEMBERED_ENTRY_BYTES` per entry. A new write over it is refused `limit_reached`, the session kept. + pub max_remembered_write_bytes: usize, +} + +impl CultMeshRudpDocumentServerOptions { + /// What one remembered write is charged: the digests of its id and content, and its answer. + pub const REMEMBERED_ENTRY_BYTES: usize = 32 + 32 + 1; } impl Default for CultMeshRudpDocumentServerOptions { @@ -193,6 +213,7 @@ impl Default for CultMeshRudpDocumentServerOptions { resend_delay: Duration::from_millis(50), max_pending_reliable_packets_per_session: 1024, max_fragment_bytes: 1200, + max_remembered_write_bytes: 32 * 1024 * 1024, } } } @@ -258,6 +279,20 @@ pub enum CultMeshRudpRejectionReason { DeleteNotSupported, /// A schema frame did not decode as a CultNet message. Carries the decoder's error text. MessageUnreadable(String), + /// A receipt-capable session sent a message id it already used for a write with other content + /// (write-receipt.md, rule 5). Nothing reached the sink; the first write's state is unchanged. + MessageIdReused, + /// A receipt-capable session already remembers `MAX_REMEMBERED_WRITES_PER_SESSION` writes and takes no new + /// id. Nothing reached the sink. + WriteLimitReached, + /// A receipt-capable session sent a message id over `MAX_MESSAGE_ID_BYTES`. Nothing was remembered. + MessageIdTooLong, + /// A receipt-capable session sent a write on an unreliable channel, where no answer could be owed once. + ChannelNotReliable, + /// A receipt-capable session's put or delete has a null or missing `document`. + DocumentMissing, + /// A receipt-capable session's put or delete has a null or missing `schemaId`. + SchemaIdMissing, } impl std::fmt::Display for CultMeshRudpRejectionReason { @@ -266,6 +301,12 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { Self::SinkRefused(error) | Self::SnapshotSourceFailed(error) => f.write_str(error), Self::DeleteNotSupported => f.write_str("the server does not take deletes"), Self::MessageUnreadable(error) => write!(f, "message could not be read: {error}"), + Self::MessageIdReused => f.write_str("the message id was already used for different content"), + Self::WriteLimitReached => f.write_str("the session remembers as many writes as it will"), + Self::MessageIdTooLong => f.write_str("the message id is longer than 256 bytes"), + Self::ChannelNotReliable => f.write_str("a receipt session's write arrived on an unreliable channel"), + Self::DocumentMissing => f.write_str("the write has no document"), + Self::SchemaIdMissing => f.write_str("the write has no schemaId"), Self::DocumentUnservable { response_bytes, max_snapshot_response_bytes, @@ -306,7 +347,7 @@ impl std::fmt::Display for CultMeshRudpRejectionReason { /// The texts a refused peer is sent, one per rejection variant. They are fixed: /// a sink's or source's error text can quote what a peer sent, so it stays with /// the caller that receives the rejection and never reaches the wire. -pub(crate) const REFUSAL_TEXTS: [&str; 11] = [ +pub(crate) const REFUSAL_TEXTS: [&str; 17] = [ "the catalog refused the document", "the document can never be served", "the snapshot source failed", @@ -318,12 +359,46 @@ pub(crate) const REFUSAL_TEXTS: [&str; 11] = [ "the snapshot response could not be sent", "the server does not take deletes", "the message could not be read", + "the message id was already used for different content", + "the session's remembered-write limit is full", + "messageId exceeds 256 bytes", + "receipt sessions require the reliable channel", + "document is required", + "schemaId is required", ]; impl CultMeshRudpRejectionReason { /// The fixed text the refused peer is sent for this variant. fn refusal_text(&self) -> &'static str { - REFUSAL_TEXTS[match self { + REFUSAL_TEXTS[usize::from(self.refusal_index())] + } + + /// The reason a remembered refusal replays as. Its payload (a sink's text, a size) is not remembered: the + /// peer is only ever sent the variant's fixed text, and the caller's rejection is reported once, at first sight. + fn from_refusal_index(index: u8) -> Self { + match index { + 0 => Self::SinkRefused(String::new()), + 1 => Self::DocumentUnservable { response_bytes: 0, max_snapshot_response_bytes: 0, fragment_count: 0, max_fragment_count: 0 }, + 2 => Self::SnapshotSourceFailed(String::new()), + 3 => Self::SnapshotTooManyDocuments { documents: 0, max_snapshot_documents: 0 }, + 4 => Self::SnapshotResponseTooLarge { response_bytes: 0, max_snapshot_response_bytes: 0 }, + 5 => Self::ResponseEncodingFailed(String::new()), + 6 => Self::PayloadBudgetFull, + 7 => Self::ResponseQueueFailed(String::new()), + 8 => Self::ResponseSendFailed(""), + 9 => Self::DeleteNotSupported, + 10 => Self::MessageUnreadable(String::new()), + 11 => Self::MessageIdReused, + 12 => Self::WriteLimitReached, + 13 => Self::MessageIdTooLong, + 14 => Self::ChannelNotReliable, + 15 => Self::DocumentMissing, + _ => Self::SchemaIdMissing, + } + } + + fn refusal_index(&self) -> u8 { + match self { Self::SinkRefused(_) => 0, Self::DocumentUnservable { .. } => 1, Self::SnapshotSourceFailed(_) => 2, @@ -335,7 +410,13 @@ impl CultMeshRudpRejectionReason { Self::ResponseSendFailed(_) => 8, Self::DeleteNotSupported => 9, Self::MessageUnreadable(_) => 10, - }] + Self::MessageIdReused => 11, + Self::WriteLimitReached => 12, + Self::MessageIdTooLong => 13, + Self::ChannelNotReliable => 14, + Self::DocumentMissing => 15, + Self::SchemaIdMissing => 16, + } } } @@ -384,7 +465,23 @@ impl CultMeshRudpApplicationRejection { /// What the refused peer is sent: a refused put or delete is a typed `write_not_authorized` naming /// the record, an unreadable frame a typed `message_invalid` naming `message`; every other rejection /// is the variant's fixed text, untyped. None carries the sink's, source's or decoder's own text. - fn refusal_message(&self) -> CultNetMessage { + /// + /// A receipt-capable session is sent `cultnet.error.v1` naming the refused message's id; any other + /// session the v0 error it has always been sent. + fn refusal_message(&self, receipts: bool) -> CultNetMessage { + let refusal = self.v0_refusal_message(); + match refusal { + CultNetMessage::Error { error, code, details } if receipts => CultNetMessage::ErrorV1 { + error, + code, + details, + message_id: Some(self.message_id.clone()).filter(|id| !id.is_empty()), + }, + other => other, + } + } + + fn v0_refusal_message(&self) -> CultNetMessage { let text = self.reason.refusal_text(); match (&self.reason, &self.record_key) { ( @@ -395,6 +492,18 @@ impl CultMeshRudpApplicationRejection { (CultMeshRudpRejectionReason::MessageUnreadable(_), _) => { message_invalid_message("message", text) } + (CultMeshRudpRejectionReason::MessageIdReused, _) => { + message_invalid_message("messageId", text) + } + (CultMeshRudpRejectionReason::MessageIdTooLong, _) => message_invalid_message("messageId", text), + (CultMeshRudpRejectionReason::DocumentMissing, _) => message_invalid_message("document", text), + (CultMeshRudpRejectionReason::SchemaIdMissing, _) => message_invalid_message("schemaId", text), + (CultMeshRudpRejectionReason::ChannelNotReliable, _) => message_invalid_message("channel", text), + (CultMeshRudpRejectionReason::WriteLimitReached, _) => CultNetMessage::Error { + error: format!("limit_reached: {text}"), + code: Some(CultNetErrorCode::LimitReached), + details: Some(CultNetErrorDetails::default()), + }, _ => CultNetMessage::Error { error: text.into(), code: None, @@ -411,6 +520,15 @@ pub enum CultMeshRudpPollOutcome { ApplicationRejected(CultMeshRudpApplicationRejection), } +/// How a refusal sent on a live session fared. +enum Answered { + Sent, + /// The refusal can never be sent as built (a permanent send failure). + Unsendable(std::io::Error), + /// The session could not encode or queue the refusal: it was not sent at all. + Unqueueable, +} + /// Why a session ends before its peer is done with it. enum SessionEnd { /// The session refused a packet. @@ -419,6 +537,24 @@ enum SessionEnd { Rejected(CultNetMessage), /// The session owes its peer a packet that can never be sent as built. Unsendable(std::io::Error), + /// The server could not answer a write it owed an answer to. Not a refusal of anything the peer sent. + LocalFailure, +} + +fn write_rejection( + session: CultMeshRudpSessionKey, + operation: CultMeshRudpApplicationOperation, + message_id: &str, + record_key: &str, + reason: CultMeshRudpRejectionReason, +) -> CultMeshRudpApplicationRejection { + CultMeshRudpApplicationRejection { + session, + operation, + message_id: message_id.to_owned(), + record_key: Some(record_key.to_owned()), + reason, + } } struct SessionEntry { @@ -431,8 +567,51 @@ struct SessionEntry { /// Set once the peer's capped frame has been answered: the session ends when the answer is acknowledged, /// or at this time. ending_deadline_monotonic_millis: Option, + /// The peer's hello listed `cultnet.write_admitted.v0` and `cultnet.error.v1`: its writes are answered + /// with a receipt or an `error.v1` naming the write's id (write-receipt.md). Framing only; it grants nothing. + receipts: bool, + /// The capability is decided once, by the session's first hello or first write, whichever comes first; + /// a later hello changes nothing. + capability_settled: bool, + /// Answered writes of a receipt-capable session: message id -> (content hash, the answer given), refusals + /// included, so a resend of the same content is answered again without being decided again and the same + /// id with other content is refused. + answered_writes: HashMap<[u8; 32], RememberedWrite>, + /// Writes refused `limit_reached`, kept so their resend replays the refusal. A bounded set outside the + /// entry budget (`MAX_REMEMBERED_WRITES_PER_SESSION`); past it a new write is refused unremembered. + limit_refusals: HashMap<[u8; 32], [u8; 32]>, + /// A limit refusal was answered that `limit_refusals` had no room to remember. That id's resend cannot be + /// told from a new write, so the session refuses every id it has not seen for the rest of its life, even once + /// the server-wide budget has freed: an id refused at a bound is never admitted later. + limit_overflowed: bool, } +/// What the server remembers of a write it answered: digests only, so an entry costs +/// `REMEMBERED_ENTRY_BYTES` whatever the id or payload size. The answer's body is rebuilt from the resend, +/// whose content hash has been checked equal to the original's. +#[derive(Clone, Copy)] +struct RememberedWrite { + content_hash: [u8; 32], + answer: StoredAnswer, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum StoredAnswer { + Admitted, + /// A refusal, by its `REFUSAL_TEXTS` index (`CultMeshRudpRejectionReason::refusal_index`). + Refused(u8), +} + +/// The longest `messageId` a receipt session may send, in UTF-8 bytes. +const MAX_MESSAGE_ID_BYTES: usize = 256; + +/// The most answered writes a receipt-capable session remembers; a session at the limit refuses new writes +/// (`WriteLimitReached`) rather than forget an id it promised to answer idempotently. +const WRITE_ADMITTED_VERSION: &str = "cultnet.write_admitted.v0"; +const ERROR_V1_VERSION: &str = "cultnet.error.v1"; + +const MAX_REMEMBERED_WRITES_PER_SESSION: usize = 1024; + /// A synchronous, multi-session CultNet RUDP raw-document server. /// /// The server owns exactly one UDP socket. Call `poll_once` from an existing @@ -444,6 +623,9 @@ pub struct CultMeshRudpDocumentServer { snapshot_source: Q, clock: C, options: CultMeshRudpDocumentServerOptions, + /// Entries remembered across all live sessions (`answered_writes`), kept in step by `remember` and + /// `drop_session`, so the server-wide budget is a comparison, not a sum over sessions per write. + remembered_writes: usize, packets_dropped: u64, send_failures: u64, /// Peers whose every datagram fails to send, standing in for an unroutable @@ -480,6 +662,7 @@ where snapshot_source, clock, options, + remembered_writes: 0, packets_dropped: 0, send_failures: 0, #[cfg(test)] @@ -570,18 +753,30 @@ where self.packets_dropped += 1; return Ok(CultMeshRudpPollOutcome::Handled); } - if packet.packet_type == CultNetRudpPacketType::Data - && packet.channel_id == "schema" - && !packet.reliable - { - return Ok(CultMeshRudpPollOutcome::Handled); + let is_schema_data = packet.packet_type == CultNetRudpPacketType::Data && packet.channel_id == "schema"; + if is_schema_data && !packet.reliable { + // An unreliable write cannot be answered once: a receipt session is refused, never applied, and + // anything else unreliable is dropped as before. Either way the peer is not idle. + if let Some(entry) = self.sessions.get_mut(&key) { + entry.last_activity_monotonic_millis = now_monotonic; + } + return self.refuse_unreliable_write(key, &packet.payload); } + // A receipt session's write on a sequenced channel (which drops superseded packets) is refused too, but + // only after the session has received and acknowledged it: an unacknowledged packet is resent forever + // and stalls every later ordered write. + let refuse_sequenced = is_schema_data && packet.sequenced && self.receipts(key); let payload_charge = if packet.packet_type == CultNetRudpPacketType::Data { packet.payload.len() } else { 0 }; if !self.payload_budget_allows(key, payload_charge) { + // A receipt-capable session's dropped write would go unanswered; ending the session answers it + // (the peer's writes settle as session_ended). Any other session is dropped from silently, as before. + if self.receipts(key) { + self.end_session(key, SessionEnd::Refused)?; + } return Ok(CultMeshRudpPollOutcome::Handled); } @@ -621,7 +816,7 @@ where }; if result.disconnected { - self.sessions.remove(&key); + self.drop_session(key); return Ok(CultMeshRudpPollOutcome::Handled); } @@ -630,6 +825,14 @@ where if frame.channel_id != "schema" { continue; } + if refuse_sequenced { + if let CultMeshRudpPollOutcome::ApplicationRejected(rejection) = + self.refuse_unreliable_write(key, &frame.payload)? + { + kept_rejection = Some(rejection); + } + continue; + } // The capped frame was answered: nothing after it is served while the answer is delivered. if self .sessions @@ -644,9 +847,12 @@ where ); // Runs of refusals this frame extends or breaks. A session is kept through a short run and // ended by `MAX_CONSECUTIVE_REFUSED_FRAMES` in a row. + // Only an undecodable frame is a protocol violation of a receipt-capable session: its refused + // deletes are refused writes, answered one by one like its refused puts. + let receipt_session = self.receipts(key); let (is_unreadable, is_delete) = ( decoded.is_err(), - matches!(decoded, Ok(CultNetMessage::DocumentDelete { .. })), + !receipt_session && matches!(decoded, Ok(CultNetMessage::DocumentDelete { .. })), ); let run = self.sessions.get_mut(&key).map(|entry| { entry.undecodable_frames_in_a_row = if is_unreadable { @@ -673,13 +879,26 @@ where now_unix, now_monotonic, )?, - Err(error) => Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::Unreadable, - message_id: String::new(), - record_key: None, - reason: CultMeshRudpRejectionReason::MessageUnreadable(format!("{error:#}")), - }), + Err(error) => { + // A receipt session's put or delete missing a required field is a refused write that + // names the field, not an unreadable frame. + let missing = if receipt_session { missing_required_write_field(&frame.payload) } else { None }; + Some(CultMeshRudpApplicationRejection { + session: key, + operation: match missing { + Some(("put", _)) => CultMeshRudpApplicationOperation::DocumentPutRaw, + Some(_) => CultMeshRudpApplicationOperation::DocumentDelete, + None => CultMeshRudpApplicationOperation::Unreadable, + }, + message_id: readable_message_id(&frame.payload).unwrap_or_default(), + record_key: None, + reason: match missing { + Some((_, "document")) => CultMeshRudpRejectionReason::DocumentMissing, + Some(_) => CultMeshRudpRejectionReason::SchemaIdMissing, + None => CultMeshRudpRejectionReason::MessageUnreadable(format!("{error:#}")), + }, + }) + } }; if let Some(rejection) = rejection { // A refused delete or unreadable frame below the cap is answered on the live session, @@ -687,19 +906,38 @@ where if (is_unreadable || is_delete) && run.is_some_and(|run| run < MAX_CONSECUTIVE_REFUSED_FRAMES) { - let refusal = rejection.refusal_message(); - let packets = match self.sessions.remove(&key) { - Some(mut entry) => { - let packets = self.refusal_packets(&mut entry.session, &refusal, false); - self.sessions.insert(key, entry); - packets + let refusal = rejection.refusal_message(self.receipts(key)); + match self.send_on_live_session(key, &refusal, self.receipts(key))? { + Answered::Sent => {} + Answered::Unsendable(error) => unsendable = Some(error), + Answered::Unqueueable => { + self.end_session(key, SessionEnd::LocalFailure)?; + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); } - None => Vec::new(), - }; - for packet in &packets { - if let Some(error) = self.send_packet(key.remote_addr, packet)? { - unsendable = Some(error); - break; + } + kept_rejection = Some(rejection); + continue; + } + // A refused write of a receipt-capable session is answered with its error.v1 and the session + // goes on: only a protocol violation (the cases above and below) ends a session. + if matches!( + rejection.operation, + CultMeshRudpApplicationOperation::DocumentPutRaw | CultMeshRudpApplicationOperation::DocumentDelete + ) && self.receipts(key) + && !matches!( + rejection.reason, + CultMeshRudpRejectionReason::ResponseSendFailed(_) + ) + { + let refusal = rejection.refusal_message(true); + match self.send_on_live_session(key, &refusal, true)? { + Answered::Sent => {} + Answered::Unsendable(error) => unsendable = Some(error), + // An unanswerable write is answered by ending the session: its peer settles it + // as `session_ended`, never as silence. + Answered::Unqueueable => { + self.end_session(key, SessionEnd::LocalFailure)?; + return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); } } kept_rejection = Some(rejection); @@ -715,7 +953,7 @@ where CultMeshRudpRejectionReason::ResponseSendFailed(_) ) { - let refusal = rejection.refusal_message(); + let refusal = rejection.refusal_message(self.receipts(key)); let packets = match self.sessions.remove(&key) { Some(mut entry) => { let packets = self.refusal_packets(&mut entry.session, &refusal, true); @@ -752,7 +990,7 @@ where rejection.reason, CultMeshRudpRejectionReason::ResponseSendFailed(_) ) { - self.end_session(key, SessionEnd::Rejected(rejection.refusal_message()))?; + self.end_session(key, SessionEnd::Rejected(rejection.refusal_message(self.receipts(key))))?; } return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)); } @@ -781,16 +1019,55 @@ where }) } + /// Refuses a put or delete that arrived on an unreliable or sequenced channel of a receipt session with + /// `message_invalid` (`details.field` `channel`) naming its id. Nothing is remembered or applied: the id + /// stays free for the same write sent reliably. + fn refuse_unreliable_write( + &mut self, + key: CultMeshRudpSessionKey, + payload: &[u8], + ) -> Result { + if !self.receipts(key) { + return Ok(CultMeshRudpPollOutcome::Handled); + } + let Ok(message) = decode_cultnet_message_from_slice(payload, CultNetWireContract::CultNetSchemaV0) else { + return Ok(CultMeshRudpPollOutcome::Handled); + }; + let (operation, message_id, record_key) = match message { + CultNetMessage::DocumentPutRaw { message_id, document } => { + (CultMeshRudpApplicationOperation::DocumentPutRaw, message_id, document.record_key) + } + CultNetMessage::DocumentDelete { message_id, record_key, .. } => { + (CultMeshRudpApplicationOperation::DocumentDelete, message_id, record_key) + } + _ => return Ok(CultMeshRudpPollOutcome::Handled), + }; + let rejection = write_rejection(key, operation, &message_id, &record_key, CultMeshRudpRejectionReason::ChannelNotReliable); + let refusal = rejection.refusal_message(true); + match self.send_on_live_session(key, &refusal, true)? { + Answered::Sent => {} + Answered::Unsendable(error) => self.end_session(key, SessionEnd::Unsendable(error))?, + Answered::Unqueueable => self.end_session(key, SessionEnd::LocalFailure)?, + } + Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection)) + } + /// Expire idle sessions and resend reliable packets whose deadline passed. pub fn maintain(&mut self) -> Result { let now = self.clock.now_monotonic_millis(); let idle_timeout_ms = duration_millis(self.options.session_idle_timeout); let lifetime_ms = duration_millis(self.options.session_max_lifetime); let before = self.sessions.len(); + let mut freed = 0; self.sessions.retain(|_, entry| { - now.saturating_sub(entry.last_activity_monotonic_millis) <= idle_timeout_ms - && now.saturating_sub(entry.created_at_monotonic_millis) <= lifetime_ms + let live = now.saturating_sub(entry.last_activity_monotonic_millis) <= idle_timeout_ms + && now.saturating_sub(entry.created_at_monotonic_millis) <= lifetime_ms; + if !live { + freed += entry.answered_writes.len(); + } + live }); + self.remembered_writes -= freed; let expired = before - self.sessions.len(); self.end_answered_sessions()?; @@ -856,7 +1133,7 @@ where .get(&key) .is_some_and(|entry| !entry.session.connect_repeats(packet)) { - self.sessions.remove(&key); + self.drop_session(key); } let reply = match self.sessions.get_mut(&key) { Some(entry) => { @@ -892,6 +1169,11 @@ where undecodable_frames_in_a_row: 0, refused_deletes_in_a_row: 0, ending_deadline_monotonic_millis: None, + receipts: false, + capability_settled: false, + answered_writes: HashMap::new(), + limit_refusals: HashMap::new(), + limit_overflowed: false, }, ); accept @@ -917,51 +1199,21 @@ where message_id, document, } => { - let record_key = document.record_key.clone(); - let reject = |reason| { - Ok(Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::DocumentPutRaw, - message_id: message_id.clone(), - record_key: Some(record_key.clone()), - reason, - })) - }; - // Admit only what some snapshot request can return: the smallest - // response that could carry this document is the record the snapshot - // source would serve for it, alone, under the shortest message id - // CultNet encodes, sized by the snapshot path's encoder and fragmented - // as the snapshot path sends it. - let served = match self.snapshot_source.served_record(&document) { - Ok(served) => served, - Err(error) => { - return reject(CultMeshRudpRejectionReason::SnapshotSourceFailed( - format!("{error:#}"), - )); - } - }; - let alone = CultNetMessage::SnapshotResponseRaw { - message_id: SHORTEST_MESSAGE_ID.into(), - documents: vec![served], - }; - let response_bytes = match encode_snapshot_response(&alone) { - Ok(payload) => payload.len(), - Err(error) => { - return reject(CultMeshRudpRejectionReason::ResponseEncodingFailed(error)); - } - }; - let fragment_count = self.fragments_for(response_bytes); - let max_fragment_count = self.max_fragments_per_response(); - if response_bytes > self.options.max_snapshot_response_bytes - || fragment_count > max_fragment_count - { - return reject(CultMeshRudpRejectionReason::DocumentUnservable { - response_bytes, - max_snapshot_response_bytes: self.options.max_snapshot_response_bytes, - fragment_count, - max_fragment_count, - }); + // The capability is fixed by the first hello or, absent one, the first write. + if let Some(entry) = self.sessions.get_mut(&key) { + entry.capability_settled = true; } + let receipts = self.receipts(key); + let record_key = document.record_key.clone(); + let rejection = |reason| { + write_rejection( + key, + CultMeshRudpApplicationOperation::DocumentPutRaw, + &message_id, + &record_key, + reason, + ) + }; let receipt = CultMeshRudpRawDocumentReceipt { session: key, message_id: message_id.clone(), @@ -969,10 +1221,86 @@ where received_at_unix_millis: now_unix, document, }; - if let Err(error) = self.sink.accept_raw_document(receipt) { - return reject(CultMeshRudpRejectionReason::SinkRefused(format!( - "{error:#}" - ))); + if !receipts { + return Ok(self.admit_put(receipt, false).err().map(rejection)); + } + let admitted = CultNetMessage::WriteAdmitted { + message_id: message_id.clone(), + operation: "put".into(), + schema_id: receipt.document.schema_id.clone(), + record_key: receipt.document.record_key.clone(), + }; + if message_id.len() > MAX_MESSAGE_ID_BYTES { + return Ok(Some(rejection(CultMeshRudpRejectionReason::MessageIdTooLong))); + } + let content_hash = match write_content_hash( + "put", + &receipt.document.schema_id, + &receipt.document.record_key, + Some(&receipt.document.payload), + ) { + Ok(hash) => hash, + Err(error) => { + return Ok(Some(rejection(CultMeshRudpRejectionReason::ResponseEncodingFailed( + format!("{error:#}"), + )))); + } + }; + // The id is looked up first: a resend is answered with what the session already said, and + // the sink is asked only about an id the session has not seen. + let id_digest = message_id_digest(&message_id); + if let ControlFlow::Break(outcome) = + self.seen_write(key, now, id_digest, content_hash, &admitted, &rejection)? + { + return Ok(outcome); + } + match self.admit_put(receipt, true) { + Ok(()) => { + // Admitted: the sink holds it. Only now is a receipt owed, never before the commit point. + self.remember(key, id_digest, content_hash, StoredAnswer::Admitted); + return self.answer_sent(key, now, &admitted, &rejection); + } + Err(reason) => { + let refused = rejection(reason); + self.remember(key, id_digest, content_hash, StoredAnswer::Refused(refused.reason.refusal_index())); + return Ok(Some(refused)); + } + } + } + CultNetMessage::Hello { + supported_message_versions, + .. + } => { + let listed = |version: &str| { + supported_message_versions + .iter() + .flatten() + .any(|candidate| candidate == version) + }; + let Some(entry) = self.sessions.get_mut(&key).filter(|entry| !entry.capability_settled) else { + return Ok(None); + }; + entry.capability_settled = true; + entry.receipts = listed(WRITE_ADMITTED_VERSION) && listed(ERROR_V1_VERSION); + if entry.receipts { + // The server's own hello tells the client this is a receipt server, not a legacy one + // that has not answered yet. Best effort: the client's window settles a missing hello. + let hello = CultNetMessage::Hello { + runtime_id: "cultmesh-rs-rudp-document-server".into(), + runtime_kind: "cultmesh-rs".into(), + agent_id: None, + role: None, + display_name: None, + supported_document_types: None, + supported_mutation_contracts: None, + supported_message_versions: Some(vec![ + WRITE_ADMITTED_VERSION.into(), + ERROR_V1_VERSION.into(), + ]), + transport_profiles: None, + supports_schema_catalog: None, + }; + self.send_reliable_schema(key, now, &hello); } } CultNetMessage::SnapshotRequest { @@ -1098,22 +1426,246 @@ where } CultNetMessage::DocumentDelete { message_id, + schema_id, record_key, - .. } => { - return Ok(Some(CultMeshRudpApplicationRejection { - session: key, - operation: CultMeshRudpApplicationOperation::DocumentDelete, - message_id, - record_key: Some(record_key), - reason: CultMeshRudpRejectionReason::DeleteNotSupported, - })); + // A delete settles the connection's capability exactly as a put does. + if let Some(entry) = self.sessions.get_mut(&key) { + entry.capability_settled = true; + } + let rejection = |reason| { + write_rejection( + key, + CultMeshRudpApplicationOperation::DocumentDelete, + &message_id, + &record_key, + reason, + ) + }; + if !self.receipts(key) { + return Ok(Some(rejection(CultMeshRudpRejectionReason::DeleteNotSupported))); + } + if message_id.len() > MAX_MESSAGE_ID_BYTES { + return Ok(Some(rejection(CultMeshRudpRejectionReason::MessageIdTooLong))); + } + let content_hash = match write_content_hash("delete", &schema_id, &record_key, None) { + Ok(hash) => hash, + Err(error) => { + return Ok(Some(rejection(CultMeshRudpRejectionReason::ResponseEncodingFailed( + format!("{error:#}"), + )))); + } + }; + let id_digest = message_id_digest(&message_id); + // A delete is never admitted; the body is the receipt it would be answered with. + let would_admit = CultNetMessage::WriteAdmitted { + message_id: message_id.clone(), + operation: "delete".into(), + schema_id: schema_id.clone(), + record_key: record_key.clone(), + }; + if let ControlFlow::Break(outcome) = + self.seen_write(key, now, id_digest, content_hash, &would_admit, &rejection)? + { + return Ok(outcome); + } + let refused = rejection(CultMeshRudpRejectionReason::DeleteNotSupported); + self.remember(key, id_digest, content_hash, StoredAnswer::Refused(refused.reason.refusal_index())); + return Ok(Some(refused)); } _ => {} } Ok(None) } + /// What a receipt-capable session already knows of a write, by the digest of its id. `Continue` is a first + /// sighting that fits both bounds. A repeat with the same content is answered with its remembered answer + /// (rebuilt from this resend, whose content hash is the original's), a repeat with other content is + /// `MessageIdReused`. A first sighting past the session table or the server-wide budget is refused + /// `limit_reached` for that write alone, and that refusal is remembered (`limit_refusals`) so its resend + /// replays it; no slot is freed, so the id is never admitted later in the session. A refusal the bounded set had no + /// room for sets `limit_overflowed`, after which the session refuses every unseen id, budget or no budget. + fn seen_write( + &mut self, + key: CultMeshRudpSessionKey, + now: u64, + id_digest: [u8; 32], + content_hash: [u8; 32], + admitted: &CultNetMessage, + rejection: &impl Fn(CultMeshRudpRejectionReason) -> CultMeshRudpApplicationRejection, + ) -> Result>> { + let Some(entry) = self.sessions.get(&key) else { + return Ok(ControlFlow::Continue(())); + }; + let prior = entry.answered_writes.get(&id_digest).copied().or_else(|| { + entry.limit_refusals.get(&id_digest).map(|&content_hash| RememberedWrite { + content_hash, + answer: StoredAnswer::Refused(CultMeshRudpRejectionReason::WriteLimitReached.refusal_index()), + }) + }); + let session_full = entry.answered_writes.len() >= MAX_REMEMBERED_WRITES_PER_SESSION; + let refusal_room = entry.limit_refusals.len() < MAX_REMEMBERED_WRITES_PER_SESSION; + let overflowed = entry.limit_overflowed; + let budget_full = self.remembered_writes + .checked_add(1) + .and_then(|entries| entries.checked_mul(CultMeshRudpDocumentServerOptions::REMEMBERED_ENTRY_BYTES)) + .is_none_or(|bytes| bytes > self.options.max_remembered_write_bytes); + Ok(ControlFlow::Break(match prior { + Some(remembered) if remembered.content_hash == content_hash => { + let answer = match remembered.answer { + StoredAnswer::Admitted => admitted.clone(), + StoredAnswer::Refused(index) => { + rejection(CultMeshRudpRejectionReason::from_refusal_index(index)).refusal_message(true) + } + }; + self.answer_sent(key, now, &answer, rejection)? + } + Some(_) => Some(rejection(CultMeshRudpRejectionReason::MessageIdReused)), + None if session_full || budget_full || overflowed => { + if let Some(entry) = self.sessions.get_mut(&key) { + if refusal_room { + entry.limit_refusals.insert(id_digest, content_hash); + } else { + entry.limit_overflowed = true; + } + } + Some(rejection(CultMeshRudpRejectionReason::WriteLimitReached)) + } + None => return Ok(ControlFlow::Continue(())), + })) + } + + fn remember(&mut self, key: CultMeshRudpSessionKey, id_digest: [u8; 32], content_hash: [u8; 32], answer: StoredAnswer) { + if let Some(entry) = self.sessions.get_mut(&key) { + if entry.answered_writes.insert(id_digest, RememberedWrite { content_hash, answer }).is_none() { + self.remembered_writes += 1; + } + } + } + + /// Removes a session for good and gives its remembered writes back to the server-wide budget. A session taken + /// out and put back within one call is not dropped. + fn drop_session(&mut self, key: CultMeshRudpSessionKey) -> Option { + let entry = self.sessions.remove(&key)?; + self.remembered_writes -= entry.answered_writes.len(); + Some(entry) + } + + /// Sends the answer owed a write. An answer that cannot be queued or sent would leave the write + /// unanswered, and a different answer would contradict what the sink did; so the session ends instead, + /// and the peer's write settles as unconfirmed (`session_ended`). The answer stays remembered. + fn answer_sent( + &mut self, + key: CultMeshRudpSessionKey, + now: u64, + answer: &CultNetMessage, + rejection: &impl Fn(CultMeshRudpRejectionReason) -> CultMeshRudpApplicationRejection, + ) -> Result> { + if self.send_reliable_schema(key, now, answer) { + return Ok(None); + } + self.end_session(key, SessionEnd::LocalFailure)?; + Ok(Some(rejection(CultMeshRudpRejectionReason::ResponseSendFailed( + "the write answer could not be sent".into(), + )))) + } + + /// Whether a put may be committed, committing it. `authorize` asks the sink first, as a receipt-capable + /// session does on an id's first sighting. + fn admit_put( + &mut self, + receipt: CultMeshRudpRawDocumentReceipt, + authorize: bool, + ) -> std::result::Result<(), CultMeshRudpRejectionReason> { + if authorize { + if let Err(error) = self.sink.authorize_raw_document(&receipt) { + return Err(CultMeshRudpRejectionReason::SinkRefused(format!("{error:#}"))); + } + } + let document = &receipt.document; + // Admit only what some snapshot request can return: the smallest + // response that could carry this document is the record the snapshot + // source would serve for it, alone, under the shortest message id + // CultNet encodes, sized by the snapshot path's encoder and fragmented + // as the snapshot path sends it. + let served = self + .snapshot_source + .served_record(document) + .map_err(|error| CultMeshRudpRejectionReason::SnapshotSourceFailed(format!("{error:#}")))?; + let alone = CultNetMessage::SnapshotResponseRaw { + message_id: SHORTEST_MESSAGE_ID.into(), + documents: vec![served], + }; + let response_bytes = encode_snapshot_response(&alone) + .map_err(CultMeshRudpRejectionReason::ResponseEncodingFailed)? + .len(); + let fragment_count = self.fragments_for(response_bytes); + let max_fragment_count = self.max_fragments_per_response(); + if response_bytes > self.options.max_snapshot_response_bytes || fragment_count > max_fragment_count { + return Err(CultMeshRudpRejectionReason::DocumentUnservable { + response_bytes, + max_snapshot_response_bytes: self.options.max_snapshot_response_bytes, + fragment_count, + max_fragment_count, + }); + } + self.sink + .accept_raw_document(receipt) + .map_err(|error| CultMeshRudpRejectionReason::SinkRefused(format!("{error:#}"))) + } + + /// Queues `message` reliably and in order on the schema channel and sends it; false when it could not be + /// encoded, queued or sent. + fn send_reliable_schema(&mut self, key: CultMeshRudpSessionKey, now: u64, message: &CultNetMessage) -> bool { + let Ok(payload) = encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0) else { + return false; + }; + let max_fragment_bytes = self.options.max_fragment_bytes; + let Some(entry) = self.sessions.get_mut(&key) else { + return false; + }; + let Ok(packets) = entry.session.send_many( + "schema", + payload, + CultNetRudpSendOptions { + reliable: true, + ordered: true, + sequenced: false, + now_ms: now, + reliable_expire_after_ms: None, + }, + Some(max_fragment_bytes), + ) else { + return false; + }; + packets + .iter() + .all(|packet| matches!(self.send_packet(key.remote_addr, packet), Ok(None))) + } + + /// Answers `refusal` on the session while it stays live. + fn send_on_live_session( + &mut self, + key: CultMeshRudpSessionKey, + refusal: &CultNetMessage, + reliable: bool, + ) -> Result { + let Some(mut entry) = self.sessions.remove(&key) else { + return Ok(Answered::Sent); + }; + let packets = self.refusal_packets(&mut entry.session, refusal, reliable); + self.sessions.insert(key, entry); + if packets.is_empty() { + return Ok(Answered::Unqueueable); + } + for packet in &packets { + if let Some(error) = self.send_packet(key.remote_addr, packet)? { + return Ok(Answered::Unsendable(error)); + } + } + Ok(Answered::Sent) + } + /// Ends a session before its peer is done with it, and says goodbye. It is /// the one way a session ends early: every end removes the session, sends /// what the peer is owed, then the goodbye. @@ -1126,13 +1678,15 @@ where /// that can never be sent as built is not dropped: the goodbye names its /// error instead. A refusal that cannot be encoded is not sent. fn end_session(&mut self, key: CultMeshRudpSessionKey, end: SessionEnd) -> Result<()> { - let Some(mut entry) = self.sessions.remove(&key) else { + let Some(mut entry) = self.drop_session(key) else { return Ok(()); }; + let local_failure = matches!(end, SessionEnd::LocalFailure); let (refusal, mut unsendable) = match end { SessionEnd::Refused => (Vec::new(), None), SessionEnd::Rejected(refusal) => (self.refusal_packets(&mut entry.session, &refusal, false), None), SessionEnd::Unsendable(error) => (Vec::new(), Some(error)), + SessionEnd::LocalFailure => (Vec::new(), None), }; entry.session.reset_peer_state(); let nothing = entry.session.create_ack(); @@ -1144,9 +1698,10 @@ where break; } } - let goodbye = match &unsendable { - Some(error) => entry.session.end_unsendable_session(error), - None => entry.session.end_refused_session(), + let goodbye = match (&unsendable, local_failure) { + (Some(error), _) => entry.session.end_unsendable_session(error), + (None, true) => entry.session.end_local_failure_session(), + (None, false) => entry.session.end_refused_session(), }; // The goodbye is the session's last datagram. If it too can never be // sent, nobody is left to tell. @@ -1154,6 +1709,10 @@ where Ok(()) } + fn receipts(&self, key: CultMeshRudpSessionKey) -> bool { + self.sessions.get(&key).is_some_and(|entry| entry.receipts) + } + /// The refusal a rejected peer is sent: unreliable and unordered, or (`reliable`) queued like any response. fn refusal_packets( &self, @@ -1265,6 +1824,9 @@ fn validate_options(options: &CultMeshRudpDocumentServerOptions) -> Result<()> { if options.session_max_lifetime.is_zero() { return Err(anyhow!("session_max_lifetime must be greater than zero")); } + if options.max_remembered_write_bytes == 0 { + return Err(anyhow!("max_remembered_write_bytes must be greater than zero")); + } if options.max_admitted_payload_bytes == 0 { return Err(anyhow!( "max_admitted_payload_bytes must be greater than zero" diff --git a/packages/cultmesh-rs/tests/rudp_document_server.rs b/packages/cultmesh-rs/tests/rudp_document_server.rs index 0e3cba66..cccff92f 100644 --- a/packages/cultmesh-rs/tests/rudp_document_server.rs +++ b/packages/cultmesh-rs/tests/rudp_document_server.rs @@ -22,6 +22,10 @@ use std::time::Duration; struct SinkState { receipts: Vec, failures_remaining: usize, + /// Record keys the sink refuses to authorize, before and at the write. + denied_keys: Vec, + /// Every `authorize_raw_document` call, by message id. + authorized: Vec, } #[derive(Clone, Default)] @@ -34,11 +38,29 @@ impl Sink { ..Default::default() }))) } + + fn denying(keys: &[&str]) -> Self { + Self(Arc::new(Mutex::new(SinkState { + denied_keys: keys.iter().map(|key| key.to_string()).collect(), + ..Default::default() + }))) + } } impl CultMeshRudpRawDocumentSink for Sink { + fn authorize_raw_document(&mut self, receipt: &CultMeshRudpRawDocumentReceipt) -> Result<()> { + self.0.lock().unwrap().authorized.push(receipt.message_id.clone()); + if self.0.lock().unwrap().denied_keys.contains(&receipt.document.record_key) { + anyhow::bail!("not authorized"); + } + Ok(()) + } + fn accept_raw_document(&mut self, receipt: CultMeshRudpRawDocumentReceipt) -> Result<()> { let mut state = self.0.lock().unwrap(); + if state.denied_keys.contains(&receipt.document.record_key) { + anyhow::bail!("not authorized"); + } if state.failures_remaining > 0 { state.failures_remaining -= 1; anyhow::bail!("injected sink failure CANARY-7f3a"); @@ -2056,3 +2078,1047 @@ fn the_grace_deadline_does_not_wrap_at_the_end_of_the_clock() -> Result<()> { assert_eq!(server.session_count(), 1); Ok(()) } + +// Write receipts (contracts/cultnet/interop/write-receipt.md). A connection whose hello lists both new frames is +// answered with `write_admitted.v0` or `error.v1`, each carrying the write's id; any other connection is answered +// exactly as before. + +fn receipt_hello() -> CultNetMessage { + CultNetMessage::Hello { + runtime_id: "receipt-client".into(), + runtime_kind: "test".into(), + agent_id: None, + role: None, + display_name: None, + supported_document_types: None, + supported_mutation_contracts: None, + supported_message_versions: Some(vec![ + "cultnet.write_admitted.v0".into(), + "cultnet.error.v1".into(), + ]), + transport_profiles: None, + supports_schema_catalog: None, + } +} + +fn put(id: &str, key: &str, payload: Vec) -> CultNetMessage { + CultNetMessage::DocumentPutRaw { + message_id: id.into(), + document: document(key, payload), + } +} + +/// Polls the server until `client` has `count` schema frames or the polling runs out; rejections are expected here. +fn frames_after( + server: &mut CultMeshRudpDocumentServer, + client: &mut CultNetRudpSocketTransportConnection, + count: usize, +) -> Result> { + let mut frames = Vec::new(); + for _ in 0..600 { + server.poll_once()?; + client.poll_resends()?; + while let Some(frame) = client.receive_once()? { + if frame.channel_id == "schema" { + frames.push(decode_cultnet_message_from_slice( + &frame.payload, + CultNetWireContract::CultNetSchemaV0, + )?); + } + } + if frames.len() >= count { + break; + } + thread::sleep(Duration::from_millis(1)); + } + Ok(frames) +} + +/// The hello a receipt server sends a receipt-capable client, ahead of any answer. +fn server_hello() -> CultNetMessage { + CultNetMessage::Hello { + runtime_id: "cultmesh-rs-rudp-document-server".into(), + runtime_kind: "cultmesh-rs".into(), + agent_id: None, + role: None, + display_name: None, + supported_document_types: None, + supported_mutation_contracts: None, + supported_message_versions: Some(vec![ + "cultnet.write_admitted.v0".into(), + "cultnet.error.v1".into(), + ]), + transport_profiles: None, + supports_schema_catalog: None, + } +} + +/// The `count` answers after the server's hello, which must come first. +fn answers_after( + server: &mut CultMeshRudpDocumentServer, + client: &mut CultNetRudpSocketTransportConnection, + count: usize, +) -> Result> { + let mut frames = frames_after(server, client, count + 1)?; + assert_eq!(frames.first(), Some(&server_hello()), "the server answers the hello first"); + Ok(frames.split_off(1)) +} + +fn admitted(id: &str, key: &str) -> CultNetMessage { + CultNetMessage::WriteAdmitted { + message_id: id.into(), + operation: "put".into(), + schema_id: "test.raw.v1".into(), + record_key: key.into(), + } +} + +#[test] +fn an_admitted_put_on_a_receipt_connection_is_answered_with_a_receipt_carrying_its_id() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 201)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + assert_eq!(answers_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + Ok(()) +} + +#[test] +fn a_connection_without_the_capability_gets_no_receipt() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 202)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![]); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "the put was admitted all the same"); + Ok(()) +} + +#[test] +fn a_resend_of_an_admitted_write_is_re_receipted_without_applying_it_twice() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 203)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + assert_eq!( + answers_after(&mut server, &mut peer, 2)?, + vec![admitted("m1", "note:1"), admitted("m1", "note:1")] + ); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + Ok(()) +} + +#[test] +fn the_same_id_with_other_content_is_refused_message_invalid_never_re_receipted() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 204)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1, 2, 3]))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![9, 9, 9]))?; + let frames = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(frames[0], admitted("m1", "note:1")); + let CultNetMessage::ErrorV1 { code, details, message_id, .. } = &frames[1] else { + panic!("the second answer must be a refusal, got {:?}", frames[1]); + }; + assert_eq!(*code, Some(CultNetErrorCode::MessageInvalid)); + assert_eq!(message_id.as_deref(), Some("m1")); + assert_eq!(details.as_ref().and_then(|d| d.field.as_deref()), Some("messageId")); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "the changed write was not applied"); + Ok(()) +} + +#[test] +fn a_refused_put_on_a_receipt_connection_is_a_typed_error_naming_its_id() -> Result<()> { + let sink = Sink::fail_once(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 205)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m7", "note:7", vec![1]))?; + let frames = answers_after(&mut server, &mut peer, 1)?; + let Some(CultNetMessage::ErrorV1 { code, message_id, .. }) = frames.first() else { + panic!("expected error.v1, got {frames:?}"); + }; + assert_eq!(*code, Some(CultNetErrorCode::WriteNotAuthorized)); + assert_eq!(message_id.as_deref(), Some("m7")); + Ok(()) +} + +#[test] +fn a_hello_listing_only_one_of_the_two_frames_does_not_opt_in() -> Result<()> { + for listed in ["cultnet.write_admitted.v0", "cultnet.error.v1"] { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 206)?; + connect(&mut server, &mut [&mut peer])?; + let mut hello = receipt_hello(); + if let CultNetMessage::Hello { supported_message_versions, .. } = &mut hello { + *supported_message_versions = Some(vec![listed.into()]); + } + send_reliable(&mut peer, &hello)?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![], "listed only {listed}"); + } + Ok(()) +} + +fn refusal_of(message: &CultNetMessage) -> (Option, Option) { + let CultNetMessage::ErrorV1 { code, message_id, .. } = message else { + panic!("expected error.v1, got {message:?}"); + }; + (code.clone(), message_id.clone()) +} + +#[test] +fn a_refused_put_leaves_the_receipt_session_open_for_the_next_write() -> Result<()> { + let sink = Sink::fail_once(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 207)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &put("m2", "note:2", vec![2]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::WriteNotAuthorized), Some("m1".into()))); + assert_eq!(answers[1], admitted("m2", "note:2")); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +#[test] +fn a_reused_id_refuses_the_write_not_the_session() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 208)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![2]))?; + send_reliable(&mut peer, &put("m3", "note:3", vec![3]))?; + let answers = answers_after(&mut server, &mut peer, 3)?; + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::MessageInvalid), Some("m1".into()))); + assert_eq!(answers[2], admitted("m3", "note:3")); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +#[test] +fn a_full_dedupe_table_refuses_new_writes_and_keeps_the_session_and_its_receipts() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 209)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..1024 { + send_reliable(&mut peer, &put(&format!("w{index}"), &format!("note:{index}"), vec![1]))?; + if index % 16 == 15 { + for _ in 0..40 { + server.poll_once()?; + peer.poll_resends()?; + while peer.receive_once()?.is_some() {} + } + } + } + send_reliable(&mut peer, &put("over", "note:over", vec![1]))?; + send_reliable(&mut peer, &put("w0", "note:0", vec![1]))?; + let answers = frames_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::LimitReached), Some("over".into()))); + let CultNetMessage::ErrorV1 { error, details, .. } = &answers[0] else { unreachable!() }; + assert_eq!(error, "limit_reached: the session's remembered-write limit is full", "the cap has its own fixed text"); + assert_eq!(details, &Some(CultNetErrorDetails::default()), "limit_reached names no field: details all null"); + assert_eq!(answers[1], admitted("w0", "note:0"), "a remembered id is still answered idempotently"); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +#[test] +fn a_readable_id_with_a_bad_field_is_refused_with_that_id() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 210)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + // A put with no document: it does not decode, but its id is there to read. + let frame = rmp_serde::to_vec_named(&serde_json::json!({ + "schemaVersion": "cultnet.document_put_raw.v0", + "messageId": "bad-1", + }))?; + peer.send_reliable("schema", frame)?; + peer.send_reliable("schema", garbage())?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::MessageInvalid), Some("bad-1".into()))); + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::MessageInvalid), None), "no id can be read"); + Ok(()) +} + +#[test] +fn a_second_hello_does_not_change_the_capability() -> Result<()> { + let mut downgrade = receipt_hello(); + if let CultNetMessage::Hello { supported_message_versions, .. } = &mut downgrade { + *supported_message_versions = None; + } + // Capable, then a hello listing nothing: still capable. + let sink = Sink::default(); + let mut server_a = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server_a.local_addr()?, 212)?; + connect(&mut server_a, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &downgrade)?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(answers_after(&mut server_a, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + // Legacy by its first write, then a capable hello: still legacy. + let mut server_b = server(Default::default(), Clock::new(70_000), Sink::default(), Source::default())?; + let mut late = client(server_b.local_addr()?, 213)?; + connect(&mut server_b, &mut [&mut late])?; + send_reliable(&mut late, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut late, &receipt_hello())?; + send_reliable(&mut late, &put("m2", "note:2", vec![1]))?; + assert_eq!(frames_after(&mut server_b, &mut late, 1)?, vec![]); + Ok(()) +} + +#[test] +fn a_receipt_session_that_runs_out_of_payload_budget_is_ended_not_left_silent() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_admitted_payload_bytes_per_session: 400, + max_snapshot_response_bytes: 200, + ..Default::default() + }; + let mut server = server(options, Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 214)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..20 { + send_reliable(&mut peer, &put(&format!("w{index}"), &format!("note:{index}"), vec![1; 64]))?; + } + frames_after(&mut server, &mut peer, usize::MAX)?; + assert_eq!(server.session_count(), 0, "the writes the budget dropped were answered by ending the session"); + Ok(()) +} + +#[test] +fn a_receipt_that_cannot_be_queued_ends_the_session_instead_of_leaving_the_write_unanswered() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_pending_reliable_packets_per_session: 3, + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 215)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + // The peer never reads, so it acknowledges nothing and the server's reliable queue fills. + for index in 0..8 { + send_reliable(&mut peer, &put(&format!("w{index}"), &format!("note:{index}"), vec![1]))?; + for _ in 0..20 { + server.poll_once()?; + } + } + assert_eq!(server.session_count(), 0, "a receipt the queue could not take ended the session"); + // The goodbye names a local failure: it is not the "session refused a packet" goodbye of a protocol refusal. + let mut reason = None; + for _ in 0..200 { + while peer.receive_once()?.is_some() {} + if let Some(found) = peer.disconnect_reason() { + reason = Some(found.to_vec()); + break; + } + thread::sleep(Duration::from_millis(1)); + } + assert_eq!(reason.expect("the peer was told the session ended"), b"server could not answer a write".to_vec()); + Ok(()) +} + +fn delete(id: &str, key: &str) -> CultNetMessage { + CultNetMessage::DocumentDelete { + message_id: id.into(), + schema_id: "test.raw.v1".into(), + record_key: key.into(), + } +} + +#[test] +fn a_resend_is_answered_from_memory_and_the_sink_is_not_asked_again() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 216)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(answers_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + // The peer's authority over the key ends; the answer already given is still the answer. + sink.0.lock().unwrap().denied_keys.push("note:1".into()); + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + Ok(()) +} + +#[test] +fn a_used_id_with_other_content_is_message_invalid_even_for_a_key_the_peer_may_not_write() -> Result<()> { + let sink = Sink::denying(&["secret"]); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 211)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &put("m1", "secret", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::MessageInvalid), Some("m1".into()))); + Ok(()) +} + +#[test] +fn a_refused_write_is_remembered_and_its_resend_replays_the_refusal() -> Result<()> { + let sink = Sink::fail_once(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 217)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::WriteNotAuthorized), Some("m1".into()))); + assert_eq!(answers[1], answers[0], "the same refusal again, not a fresh attempt that would now succeed"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0); + Ok(()) +} + +/// The sink is asked once, on an id's first sighting: not again for a resend (admitted or refused), and +/// not at all on a session that never declared the receipt capability. The sink here only counts, so a +/// flag that stopped or inverted the ask cannot hide behind the sink refusing at the write. +#[test] +fn authorize_is_called_once_per_first_sighting_and_only_for_receipt_sessions() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 219)?; + let mut legacy = client(server.local_addr()?, 220)?; + connect(&mut server, &mut [&mut peer, &mut legacy])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(answers_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + assert_eq!(sink.0.lock().unwrap().authorized, vec!["m1".to_string()], "first sighting asks once"); + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + assert_eq!(sink.0.lock().unwrap().authorized, vec!["m1".to_string()], "the resend does not ask again"); + send_reliable(&mut legacy, &put("l1", "note:9", vec![1]))?; + for _ in 0..40 { + server.poll_once()?; + legacy.poll_resends()?; + while legacy.receive_once()?.is_some() {} + } + let state = sink.0.lock().unwrap(); + assert_eq!(state.receipts.len(), 2, "the legacy put was committed"); + assert_eq!(state.authorized, vec!["m1".to_string()], "a legacy session is never asked"); + Ok(()) +} + +#[test] +fn a_delete_that_reuses_a_puts_id_is_message_invalid() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 218)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + send_reliable(&mut peer, &delete("m1", "note:1"))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(answers[0], admitted("m1", "note:1")); + let CultNetMessage::ErrorV1 { code, details, message_id, .. } = &answers[1] else { + panic!("expected error.v1, got {:?}", answers[1]); + }; + assert_eq!(*code, Some(CultNetErrorCode::MessageInvalid)); + assert_eq!(message_id.as_deref(), Some("m1")); + assert_eq!(details.as_ref().and_then(|d| d.field.as_deref()), Some("messageId")); + Ok(()) +} + +#[test] +fn deletes_on_a_receipt_session_are_refused_one_by_one_and_never_end_it() -> Result<()> { + let mut server = server(Default::default(), Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 219)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..12 { + send_reliable(&mut peer, &delete(&format!("d{index}"), "note:1"))?; + } + send_reliable(&mut peer, &delete("d3", "note:1"))?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 14)?; + assert_eq!(answers.len(), 14); + assert_eq!(refusal_of(&answers[12]), refusal_of(&answers[3]), "a resent delete replays its refusal"); + assert_eq!(answers[13], admitted("m1", "note:1")); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +#[test] +fn a_delete_as_the_first_frame_settles_the_connection_as_legacy() -> Result<()> { + let mut server = server(Default::default(), Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 220)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &delete("d1", "note:1"))?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + let frames = frames_after(&mut server, &mut peer, 3)?; + assert_eq!(frames.len(), 1, "only the legacy delete refusal: {frames:?}"); + assert!(matches!(frames[0], CultNetMessage::Error { .. }), "{:?}", frames[0]); + Ok(()) +} + +#[test] +fn a_refusal_that_cannot_be_queued_ends_the_session_instead_of_vanishing() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_pending_reliable_packets_per_session: 3, + ..Default::default() + }; + let mut server = server(options, Clock::new(70_000), Sink::denying(&["x"]), Source::default())?; + let mut peer = client(server.local_addr()?, 221)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..8 { + send_reliable(&mut peer, &put(&format!("w{index}"), "x", vec![1]))?; + for _ in 0..20 { + server.poll_once()?; + } + } + assert_eq!(server.session_count(), 0, "a refusal the queue could not take ended the session"); + Ok(()) +} + +#[test] +fn refusal_frames_from_a_peer_are_not_protocol_violations() -> Result<()> { + let mut server = server(Default::default(), Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 222)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for _ in 0..12 { + send_reliable( + &mut peer, + &CultNetMessage::ErrorV1 { + error: "message_invalid: nope".into(), + code: Some(CultNetErrorCode::MessageInvalid), + details: None, + message_id: Some("x".into()), + }, + )?; + send_reliable( + &mut peer, + &CultNetMessage::Error { error: "nope".into(), code: None, details: None }, + )?; + } + send_reliable(&mut peer, &put("m1", "note:1", vec![1]))?; + assert_eq!(answers_after(&mut server, &mut peer, 1)?, vec![admitted("m1", "note:1")]); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +#[test] +fn a_put_that_reuses_a_deletes_id_is_message_invalid() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 223)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &delete("d1", "note:1"))?; + send_reliable(&mut peer, &put("d1", "note:1", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::MessageInvalid), Some("d1".into()))); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "the reusing put was not applied"); + Ok(()) +} + +fn error_text_and_field(message: &CultNetMessage) -> (String, Option) { + let CultNetMessage::ErrorV1 { error, details, .. } = message else { panic!("expected error.v1, got {message:?}") }; + (error.clone(), details.as_ref().and_then(|d| d.field.clone())) +} + +/// The server-wide budget counts fixed-size entries across sessions; a write over it is refused for that write +/// alone, the refusal is remembered (a resend replays it, no slot is freed), and every session is kept. +#[test] +fn a_server_wide_memory_budget_refuses_the_write_over_it_and_keeps_the_sessions() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 2 * 65, + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(70_000), sink.clone(), Source::default())?; + let mut a = client(server.local_addr()?, 230)?; + let mut b = client(server.local_addr()?, 231)?; + connect(&mut server, &mut [&mut a, &mut b])?; + send_reliable(&mut a, &receipt_hello())?; + send_reliable(&mut b, &receipt_hello())?; + send_reliable(&mut a, &put("a1", "note:a1", vec![1]))?; + send_reliable(&mut b, &put("b1", "note:b1", vec![1]))?; + for _ in 0..200 { + server.poll_once()?; + a.poll_resends()?; + b.poll_resends()?; + while a.receive_once()?.is_some() {} + while b.receive_once()?.is_some() {} + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 2); + send_reliable(&mut a, &put("a2", "note:a2", vec![1]))?; + send_reliable(&mut a, &put("a2", "note:a2", vec![1]))?; + let answers = frames_after(&mut server, &mut a, 2)?; + for answer in &answers { + assert_eq!(refusal_of(answer), (Some(CultNetErrorCode::LimitReached), Some("a2".into())), "the resend replays the refusal"); + let CultNetMessage::ErrorV1 { details, .. } = answer else { unreachable!() }; + assert_eq!(details, &Some(CultNetErrorDetails::default())); + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 2, "the refused write was not applied"); + assert_eq!(server.session_count(), 2, "both sessions are kept"); + send_reliable(&mut a, &put("a1", "note:a1", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut a, 1)?, vec![admitted("a1", "note:a1")], "a remembered id is still answered"); + Ok(()) +} + +/// A message id over 256 bytes is refused before anything is remembered: no entry, no budget charge. +#[test] +fn a_message_id_over_256_bytes_is_refused_before_anything_is_remembered() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 65, + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 232)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + let long = "x".repeat(257); + send_reliable(&mut peer, &put(&long, "note:1", vec![1]))?; + send_reliable(&mut peer, &put(&long, "note:1", vec![1]))?; + send_reliable(&mut peer, &put(&"y".repeat(256), "note:2", vec![1]))?; + let answers = answers_after(&mut server, &mut peer, 3)?; + for answer in &answers[..2] { + assert_eq!(refusal_of(answer), (Some(CultNetErrorCode::MessageInvalid), Some(long.clone()))); + assert_eq!(error_text_and_field(answer), ("message_invalid: messageId exceeds 256 bytes".into(), Some("messageId".into()))); + } + assert_eq!(answers[2], admitted(&"y".repeat(256), "note:2"), "256 bytes is allowed, and the long id charged no budget"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "the long-id writes were never applied"); + Ok(()) +} + +/// A write on an unreliable channel of a receipt session is refused, not dropped and not accepted. +#[test] +fn an_unreliable_write_on_a_receipt_session_is_refused_message_invalid_naming_the_channel() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 233)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let payload = encode_cultnet_message_to_vec(&put("u1", "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: false, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + let mut answers = Vec::new(); + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + answers.push(decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?); + } + } + let refusals: Vec<_> = answers.iter().filter(|answer| matches!(answer, CultNetMessage::ErrorV1 { .. })).collect(); + assert_eq!(refusals.len(), 1, "the unreliable write is answered, not dropped: {answers:?}"); + assert_eq!(refusal_of(refusals[0]), (Some(CultNetErrorCode::MessageInvalid), Some("u1".into()))); + assert_eq!( + error_text_and_field(refusals[0]), + ("message_invalid: receipt sessions require the reliable channel".into(), Some("channel".into())) + ); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "nothing was applied"); + assert_eq!(server.session_count(), 1); + // The id was not consumed: the same write on the reliable channel is admitted. + peer.send(&put("u1", "note:1", vec![1]))?; + server.poll_once()?; + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1); + Ok(()) +} + +/// A limit refusal is remembered like any refusal: the same id with other content is a reused id, not a fresh +/// limit refusal; and the remembered refusals are a bounded set (1024), past which a refusal is not remembered. +#[test] +fn limit_refusals_are_remembered_in_a_bounded_set() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 65, + ..Default::default() + }; + let mut server = server(options, Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = client(server.local_addr()?, 234)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + send_reliable(&mut peer, &put("full", "note:full", vec![1]))?; + for index in 0..1024 { + send_reliable(&mut peer, &put(&format!("r{index}"), "note:r", vec![1]))?; + if index % 16 == 15 { + for _ in 0..40 { + server.poll_once()?; + peer.poll_resends()?; + while peer.receive_once()?.is_some() {} + } + } + } + // r0 was remembered as refused: other content under its id is a reused id. + send_reliable(&mut peer, &put("r0", "note:other", vec![2]))?; + let answers = frames_after(&mut server, &mut peer, 1)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::MessageInvalid), Some("r0".into()))); + // The 1025th refusal found the set full: it was refused but not remembered. + send_reliable(&mut peer, &put("over", "note:over", vec![1]))?; + send_reliable(&mut peer, &put("over", "note:over", vec![2]))?; + let answers = frames_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::LimitReached), Some("over".into()))); + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::LimitReached), Some("over".into())), "not remembered: refused afresh"); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +/// A refusal the bounded set could not hold is still the session's first answer for that id: when the budget +/// frees (another session ends), the id is not admitted later in the session, and neither is any id the session +/// has not been seen to use, because the refusal that was not remembered cannot be told from a new write. +#[test] +fn a_session_whose_limit_refusals_overflowed_refuses_new_ids_after_the_budget_frees() -> Result<()> { + let clock = Clock::new(70_000); + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 65, + session_max_lifetime: Duration::from_millis(100), + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, clock.clone(), sink.clone(), Source::default())?; + let mut holder = client(server.local_addr()?, 240)?; + connect(&mut server, &mut [&mut holder])?; + send_reliable(&mut holder, &receipt_hello())?; + send_reliable(&mut holder, &put("held", "note:held", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut holder, 2)?[1], admitted("held", "note:held"), "after the server's hello"); + + clock.set_monotonic(70_050); + let mut peer = client(server.local_addr()?, 241)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + for index in 0..1024 { + send_reliable(&mut peer, &put(&format!("r{index}"), "note:r", vec![1]))?; + if index % 16 == 15 { + for _ in 0..40 { + server.poll_once()?; + peer.poll_resends()?; + while peer.receive_once()?.is_some() {} + } + } + } + send_reliable(&mut peer, &put("over", "note:over", vec![1]))?; + let answers = frames_after(&mut server, &mut peer, 1)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::LimitReached), Some("over".into()))); + + clock.set_monotonic(70_101); + assert_eq!(server.maintain()?.sessions_expired, 1, "the holder ended and freed the budget"); + assert_eq!(server.session_count(), 1); + send_reliable(&mut peer, &put("over", "note:over", vec![1]))?; + send_reliable(&mut peer, &put("fresh", "note:fresh", vec![1]))?; + let answers = frames_after(&mut server, &mut peer, 2)?; + assert_eq!(refusal_of(&answers[0]), (Some(CultNetErrorCode::LimitReached), Some("over".into())), "never admitted later"); + assert_eq!(refusal_of(&answers[1]), (Some(CultNetErrorCode::LimitReached), Some("fresh".into()))); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 1, "only the holder's write was applied"); + Ok(()) +} + +/// A session that ends gives its remembered entries back to the server-wide budget, whichever way it ends. +#[test] +fn a_session_that_ends_frees_its_remembered_writes_from_the_budget() -> Result<()> { + let clock = Clock::new(70_000); + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 65, + session_max_lifetime: Duration::from_millis(100), + ..Default::default() + }; + let sink = Sink::default(); + let mut server = server(options, clock.clone(), sink.clone(), Source::default())?; + let mut first = client(server.local_addr()?, 242)?; + connect(&mut server, &mut [&mut first])?; + send_reliable(&mut first, &receipt_hello())?; + send_reliable(&mut first, &put("a", "note:a", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut first, 2)?[1], admitted("a", "note:a"), "after the server's hello"); + + clock.set_monotonic(70_050); + let mut second = client(server.local_addr()?, 243)?; + connect(&mut server, &mut [&mut second])?; + send_reliable(&mut second, &receipt_hello())?; + send_reliable(&mut second, &put("b", "note:b", vec![1]))?; + assert_eq!(refusal_of(&frames_after(&mut server, &mut second, 2)?[1]).0, Some(CultNetErrorCode::LimitReached)); + + clock.set_monotonic(70_101); + assert_eq!(server.maintain()?.sessions_expired, 1); + send_reliable(&mut second, &put("c", "note:c", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut second, 1)?, vec![admitted("c", "note:c")], "the ended session's entry was freed"); + clock.set_monotonic(70_152); + assert_eq!(server.maintain()?.sessions_expired, 1); + assert_eq!(server.session_count(), 0); + let mut third = client(server.local_addr()?, 244)?; + connect(&mut server, &mut [&mut third])?; + send_reliable(&mut third, &receipt_hello())?; + send_reliable(&mut third, &put("d", "note:d", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut third, 2)?[1], admitted("d", "note:d"), "and so was the second's"); + Ok(()) +} + +/// Whichever way a session ends, its remembered writes go back to the budget: here the peer says goodbye, and +/// here the server ends it itself (an unreliable write whose refusal the reliable queue cannot take). +#[test] +fn a_disconnected_or_server_ended_session_frees_its_remembered_writes() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { + max_remembered_write_bytes: 65, + max_pending_reliable_packets_per_session: 3, + ..Default::default() + }; + let mut server = server(options, Clock::new(70_000), Sink::default(), Source::default())?; + let mut first = client(server.local_addr()?, 245)?; + connect(&mut server, &mut [&mut first])?; + send_reliable(&mut first, &receipt_hello())?; + send_reliable(&mut first, &put("a", "note:a", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut first, 2)?[1], admitted("a", "note:a")); + first.disconnect(Vec::new())?; + for _ in 0..20 { + server.poll_once()?; + } + assert_eq!(server.session_count(), 0, "the goodbye ended the session"); + + // The budget holds one entry. A raw peer is admitted into it, then ended by the server. + let mut second = RawPeer::connect(&mut server, 246)?; + second.send(&receipt_hello())?; + second.send(&put("b", "note:b", vec![1]))?; + server.poll_once()?; + for index in 0..8 { + let payload = encode_cultnet_message_to_vec(&put(&format!("u{index}"), "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = second.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: false, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + second.send_packet(packet)?; + } + server.poll_once()?; + } + assert_eq!(server.session_count(), 0, "the server ended the session"); + + let mut third = client(server.local_addr()?, 247)?; + connect(&mut server, &mut [&mut third])?; + send_reliable(&mut third, &receipt_hello())?; + send_reliable(&mut third, &put("c", "note:c", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut third, 2)?[1], admitted("c", "note:c"), "both ended sessions' entries were freed"); + Ok(()) +} + +/// Receipts need a channel that is reliable and not sequenced: a reliable-sequenced write is refused like an +/// unreliable one, a reliable-unordered write is admitted, and a legacy session's sequenced write is left alone. +#[test] +fn only_reliable_non_sequenced_writes_are_accepted_on_a_receipt_session() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 248)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + let send_with = |peer: &mut RawPeer, server: &mut Server, id: &str, ordered: bool, sequenced: bool| -> Result> { + let payload = encode_cultnet_message_to_vec(&put(id, "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: true, ordered, sequenced, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + let mut answers = Vec::new(); + for packet in peer.drain() { + for frame in peer.session.receive(&packet, 4)?.delivered { + answers.push(decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0)?); + } + } + Ok(answers) + }; + let sequenced = send_with(&mut peer, &mut server, "s1", false, true)?; + let refusals: Vec<_> = sequenced.iter().filter(|a| matches!(a, CultNetMessage::ErrorV1 { .. })).collect(); + assert_eq!(refusals.len(), 1, "the sequenced write is answered: {sequenced:?}"); + assert_eq!(refusal_of(refusals[0]), (Some(CultNetErrorCode::MessageInvalid), Some("s1".into()))); + assert_eq!( + error_text_and_field(refusals[0]), + ("message_invalid: receipt sessions require the reliable channel".into(), Some("channel".into())) + ); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0, "nothing was applied"); + + let unordered = send_with(&mut peer, &mut server, "s1", false, false)?; + assert!(unordered.contains(&admitted("s1", "note:1")), "reliable unordered is accepted, and the id was not consumed: {unordered:?}"); + // The same peer: the refused packet was acknowledged, so an ordered stream does not wait on its sequence number. + let ordered = send_with(&mut peer, &mut server, "o1", true, false)?; + assert!(ordered.contains(&admitted("o1", "note:1")), "reliable ordered is accepted: {ordered:?}"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 2); + + let legacy_sink = Sink::default(); + let mut legacy_server: Server = CultMeshRudpDocumentServer::new( + UdpSocket::bind("127.0.0.1:0")?, + legacy_sink.clone(), + Source::default(), + Clock::new(70_000), + Default::default(), + )?; + let mut legacy = RawPeer::connect(&mut legacy_server, 249)?; + let none = send_with(&mut legacy, &mut legacy_server, "l1", false, true)?; + assert!(none.is_empty(), "a legacy session is sent nothing: {none:?}"); + assert_eq!(legacy_sink.0.lock().unwrap().receipts.len(), 1, "and its reliable-sequenced write is applied as before"); + Ok(()) +} + +/// A refused write is traffic: the peer that sends it is not idle, so its session outlives the idle timeout measured +/// from the refusal and not from the packet before it. +#[test] +fn a_refused_write_keeps_the_session_from_going_idle() -> Result<()> { + let clock = Clock::new(70_000); + let options = CultMeshRudpDocumentServerOptions { session_idle_timeout: Duration::from_secs(1), ..Default::default() }; + let mut server = server(options, clock.clone(), Sink::default(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 252)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + for (at, reliable, sequenced) in [(70_700, false, false), (71_400, true, true)] { + clock.set(at); + let payload = encode_cultnet_message_to_vec(&put("idle", "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable, ordered: false, sequenced, now_ms: at, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + assert_eq!(server.maintain()?.sessions_expired, 0, "the refused write at {at} is activity"); + } + assert_eq!(server.session_count(), 1); + Ok(()) +} + +/// A Connect that does not repeat the accepted one is a new client session: it starts with nothing remembered, so +/// the id a receipt session already used is a new write on it, and the old session's entries leave the budget. +#[test] +fn a_reconnect_is_a_new_receipt_session_that_remembers_nothing_of_the_old() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { max_remembered_write_bytes: 65, ..Default::default() }; + let sink = Sink::default(); + let mut server = server(options, Clock::new(70_000), sink.clone(), Source::default())?; + let target = server.local_addr()?; + let shared = UdpSocket::bind("127.0.0.1:0")?; + let twin = shared.try_clone()?; + let mut first = client_on(shared, target, 250, 50_000)?; + connect(&mut server, &mut [&mut first])?; + send_reliable(&mut first, &receipt_hello())?; + send_reliable(&mut first, &put("m1", "note:a", vec![1]))?; + assert_eq!(frames_after(&mut server, &mut first, 2)?[1], admitted("m1", "note:a")); + + let mut restarted = client_on(twin, target, 250, 7)?; + connect(&mut server, &mut [&mut restarted])?; + assert_eq!(server.session_count(), 1); + send_reliable(&mut restarted, &receipt_hello())?; + send_reliable(&mut restarted, &put("m1", "note:b", vec![2]))?; + let answers = frames_after(&mut server, &mut restarted, 2)?; + assert_eq!(answers[1], admitted("m1", "note:b"), "same id, other content: a new write on a new session, within a one-entry budget"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 2); + Ok(()) +} + +/// A session that never declared the capability gets no refusal for an unreliable write: it is dropped as before. +#[test] +fn an_unreliable_write_on_a_legacy_session_is_dropped_silently() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 235)?; + let payload = encode_cultnet_message_to_vec(&put("u1", "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: false, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + assert!(peer.drain().is_empty(), "a legacy session is sent nothing"); + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0); + Ok(()) +} + +/// A put or delete whose document or schemaId is null or missing is refused `message_invalid` naming that +/// field (document first), with the spec's fixed text and the write's id; the session goes on. +#[test] +fn a_write_missing_its_document_or_schema_id_is_refused_naming_the_field() -> Result<()> { + let sink = Sink::default(); + let mut server = server(Default::default(), Clock::new(70_000), sink.clone(), Source::default())?; + let mut peer = client(server.local_addr()?, 236)?; + connect(&mut server, &mut [&mut peer])?; + send_reliable(&mut peer, &receipt_hello())?; + let frames = [ + ("p-missing", serde_json::json!({"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "p-missing"}), "document"), + ("p-null", serde_json::json!({"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "p-null", "document": null}), "document"), + ("p-noschema", serde_json::json!({"schemaVersion": "cultnet.document_put_raw.v0", "messageId": "p-noschema", + "document": {"recordKey": "k", "storedAt": "t", "payloadEncoding": "messagepack", "payload": [1]}}), "schemaId"), + ("d-null", serde_json::json!({"schemaVersion": "cultnet.document_delete.v0", "messageId": "d-null", "schemaId": null, "recordKey": "k"}), "schemaId"), + ("d-missing", serde_json::json!({"schemaVersion": "cultnet.document_delete.v0", "messageId": "d-missing", "recordKey": "k"}), "schemaId"), + ]; + for (_, frame, _) in &frames { + peer.send_reliable("schema", rmp_serde::to_vec_named(frame)?)?; + } + let answers = answers_after(&mut server, &mut peer, frames.len())?; + for ((id, _, field), answer) in frames.iter().zip(&answers) { + assert_eq!(refusal_of(answer), (Some(CultNetErrorCode::MessageInvalid), Some((*id).into()))); + assert_eq!( + error_text_and_field(answer), + (format!("message_invalid: {field} is required"), Some((*field).into())) + ); + } + assert_eq!(sink.0.lock().unwrap().receipts.len(), 0); + assert_eq!(server.session_count(), 1); + Ok(()) +} + +/// An unreliable write whose refusal the full reliable queue cannot take ends the session as a local failure +/// (the peer's write is then unconfirmed, never silent and never "refused a packet"). +#[test] +fn an_unreliable_refusal_that_cannot_be_queued_ends_the_session_as_a_local_failure() -> Result<()> { + let options = CultMeshRudpDocumentServerOptions { max_pending_reliable_packets_per_session: 3, ..Default::default() }; + let mut server = server(options, Clock::new(70_000), Sink::default(), Source::default())?; + let mut peer = RawPeer::connect(&mut server, 237)?; + peer.send(&receipt_hello())?; + server.poll_once()?; + // The peer acknowledges nothing, so each refusal stays in the server's reliable queue. + for index in 0..8 { + let payload = encode_cultnet_message_to_vec(&put(&format!("u{index}"), "note:1", vec![1]), CultNetWireContract::CultNetSchemaV0)?; + let packets = peer.session.send_many( + "schema", + payload, + cultnet_rs::CultNetRudpSendOptions { reliable: false, ordered: false, sequenced: false, now_ms: 3, reliable_expire_after_ms: None }, + Some(1200), + )?; + for packet in &packets { + peer.send_packet(packet)?; + } + server.poll_once()?; + } + assert_eq!(server.session_count(), 0, "the refusal the queue could not take ended the session"); + let goodbye = peer.drain().into_iter().find(is_goodbye).expect("the peer is told the session ended"); + assert_eq!(goodbye.payload, b"server could not answer a write".to_vec()); + Ok(()) +} diff --git a/packages/cultnet-rs/src/contracts.rs b/packages/cultnet-rs/src/contracts.rs index ebc32afa..0ea2aa53 100644 --- a/packages/cultnet-rs/src/contracts.rs +++ b/packages/cultnet-rs/src/contracts.rs @@ -249,6 +249,9 @@ pub enum CultNetErrorCode { WriteNotAuthorized, UnsupportedSchemaVersion, MessageInvalid, + /// `cultnet.error.v1` only: the session's remembered-write limit is full. `cultnet.error.v0` neither + /// encodes nor decodes it, so its enum stays the schema's. + LimitReached, } impl CultNetErrorCode { @@ -263,6 +266,7 @@ impl CultNetErrorCode { Self::WriteNotAuthorized => "write_not_authorized", Self::UnsupportedSchemaVersion => "unsupported_schema_version", Self::MessageInvalid => "message_invalid", + Self::LimitReached => "limit_reached", } } @@ -277,6 +281,7 @@ impl CultNetErrorCode { "write_not_authorized" => Ok(Self::WriteNotAuthorized), "unsupported_schema_version" => Ok(Self::UnsupportedSchemaVersion), "message_invalid" => Ok(Self::MessageInvalid), + "limit_reached" => Ok(Self::LimitReached), _ => Err(anyhow!("ErrorMessage.Code is not a recognized cultnet.error.v0 code")), } } @@ -359,6 +364,26 @@ pub enum CultNetMessage { #[serde(default, skip_serializing_if = "Option::is_none")] details: Option, }, + /// `cultnet.error.v1` (contracts/cultnet/interop/write-receipt.md): `cultnet.error.v0` plus the id of the + /// request it answers. Sent only to a connection that declared the write-receipt capability. + #[serde(skip_deserializing)] + #[serde(rename = "cultnet.error.v1", rename_all = "camelCase")] + ErrorV1 { + error: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + code: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + details: Option, + message_id: Option, + }, + /// `cultnet.write_admitted.v0`: the receipt for one admitted put or delete, sent after its commit point. + #[serde(rename = "cultnet.write_admitted.v0", rename_all = "camelCase")] + WriteAdmitted { + message_id: String, + operation: String, + schema_id: String, + record_key: String, + }, #[serde(rename = "cultnet.sample.change_name.v0", rename_all = "camelCase")] SampleChangeName { name: String }, #[serde(rename = "cultnet.sample.chat.v0", rename_all = "camelCase")] @@ -543,7 +568,9 @@ pub fn encode_cultnet_message_for_wire( | CultNetMessage::SnapshotResponseRawV1 { .. } | CultNetMessage::ContentChunkRequest { .. } | CultNetMessage::ContentChunkResponse { .. } - | CultNetMessage::Error { .. } => encode_raw_cultnet_schema_message(message), + | CultNetMessage::Error { .. } + | CultNetMessage::ErrorV1 { .. } + | CultNetMessage::WriteAdmitted { .. } => encode_raw_cultnet_schema_message(message), _ => Ok(cultcache_rs::decode_payload(&rmp_serde::to_vec( &serde_json::to_value(message)?, )?)?), @@ -573,7 +600,8 @@ pub fn decode_cultnet_message_from_slice( | "cultnet.snapshot_response_raw.v0" | "cultmesh.content_chunk_request.v1" | "cultmesh.content_chunk_response.v1" - | "cultnet.error.v0", + | "cultnet.error.v0" + | "cultnet.error.v1", ) => parse_raw_cultnet_schema_message(&wire_value) .with_context(|| received_version_label(&wire_value))?, _ => cultcache_rs::decode_payload::(bytes) @@ -645,6 +673,20 @@ fn validate_message(message: &CultNetMessage) -> Result<()> { require_non_empty(session, "session")?; } CultNetMessage::Error { error, code: _, details: _ } => require_non_empty(error, "error")?, + CultNetMessage::ErrorV1 { error, message_id, .. } => { + require_non_empty(error, "error")?; + if let Some(message_id) = message_id { + require_non_empty(message_id, "messageId")?; + } + } + CultNetMessage::WriteAdmitted { message_id, operation, schema_id, record_key } => { + require_non_empty(message_id, "messageId")?; + if operation != "put" && operation != "delete" { + return Err(anyhow!("operation must be put or delete")); + } + require_non_empty(schema_id, "schemaId")?; + require_non_empty(record_key, "recordKey")?; + } CultNetMessage::SampleChangeName { name } => require_non_empty(name, "name")?, CultNetMessage::SampleChat { text } => require_non_empty(text, "text")?, CultNetMessage::DocumentDelete { @@ -1021,7 +1063,7 @@ fn parse_raw_cultnet_schema_message(input: &rmpv::Value) -> Result { + "cultnet.error.v0" | "cultnet.error.v1" => { let code = get("code") .filter(|value| !value.is_nil()) .map(|value| { @@ -1031,12 +1073,23 @@ fn parse_raw_cultnet_schema_message(input: &rmpv::Value) -> Result None, + other => Some(require_legacy_string(other, "messageId")?), + }, + }); + } + Ok(CultNetMessage::Error { error, code, details }) } _ => Err(anyhow!( "Unsupported raw cultnet.schema.v0 message" @@ -1090,7 +1143,18 @@ fn require_legacy_optional_u64( } fn encode_raw_cultnet_schema_message(message: &CultNetMessage) -> Result { + if let CultNetMessage::Error { code: Some(CultNetErrorCode::LimitReached), .. } = message { + return Err(anyhow!("limit_reached is a cultnet.error.v1 code; cultnet.error.v0 cannot carry it")); + } Ok(rmpv::Value::Map(match message { + // Spec field order (write-receipt.md), not serde_json's sorted map. + CultNetMessage::WriteAdmitted { message_id, operation, schema_id, record_key } => vec![ + (rmpv::Value::from("schemaVersion"), rmpv::Value::from("cultnet.write_admitted.v0")), + (rmpv::Value::from("messageId"), rmpv::Value::from(message_id.as_str())), + (rmpv::Value::from("operation"), rmpv::Value::from(operation.as_str())), + (rmpv::Value::from("schemaId"), rmpv::Value::from(schema_id.as_str())), + (rmpv::Value::from("recordKey"), rmpv::Value::from(record_key.as_str())), + ], CultNetMessage::DocumentPutRaw { message_id, document, @@ -1255,6 +1319,36 @@ fn encode_raw_cultnet_schema_message(message: &CultNetMessage) -> Result vec![ + (rmpv::Value::from("schemaVersion"), rmpv::Value::from("cultnet.error.v1")), + (rmpv::Value::from("error"), rmpv::Value::from(error.as_str())), + (rmpv::Value::from("routingHint"), rmpv::Value::Nil), + ( + rmpv::Value::from("code"), + code.as_ref() + .map(|code| rmpv::Value::from(code.as_wire_str())) + .unwrap_or(rmpv::Value::Nil), + ), + ( + rmpv::Value::from("details"), + details + .as_ref() + .map(encode_error_details) + .unwrap_or(rmpv::Value::Nil), + ), + ( + rmpv::Value::from("messageId"), + message_id + .as_deref() + .map(rmpv::Value::from) + .unwrap_or(rmpv::Value::Nil), + ), + ], _ => { return Err(anyhow!( "Message is not a raw cultnet.schema.v0 binary replication message" @@ -1884,3 +1978,71 @@ fn legacy_schema_descriptor(schema: &CultNetSchemaDescriptor) -> Result, +) -> Result<[u8; 32]> { + use sha2::{Digest, Sha256}; + let value = rmpv::Value::Array(vec![ + rmpv::Value::from(operation), + rmpv::Value::from(schema_id), + rmpv::Value::from(record_key), + payload.map_or(rmpv::Value::Nil, |payload| rmpv::Value::Binary(payload.to_vec())), + ]); + let mut bytes = Vec::new(); + rmpv::encode::write_value(&mut bytes, &value)?; + Ok(Sha256::digest(&bytes).into()) +} + +/// SHA-256 of a write's `messageId`: what a receipt server remembers in place of the id itself, so an +/// entry's memory does not depend on the id's length (write-receipt.md, "What is remembered"). +pub fn message_id_digest(message_id: &str) -> [u8; 32] { + use sha2::{Digest, Sha256}; + Sha256::digest(message_id.as_bytes()).into() +} + +/// The `messageId` of a schema frame that does not decode as a message (write-receipt.md, rule 3: a refusal +/// carries the write's id whenever the frame is readable enough to find it). `None` when the bytes are not +/// a MessagePack map or hold no non-empty string `messageId`. +pub fn readable_message_id(bytes: &[u8]) -> Option { + let value = rmpv::decode::read_value(&mut &*bytes).ok()?; + let rmpv::Value::Map(entries) = value else { + return None; + }; + entries.into_iter().find_map(|(key, value)| match (key.as_str(), value) { + (Some("messageId"), rmpv::Value::String(id)) => id.into_str().filter(|id| !id.is_empty()), + _ => None, + }) +} + +/// The first required field (`document`, then `schemaId`) that a put or delete frame lacks or holds as nil +/// (write-receipt.md, "Required fields"), with the write's operation (`"put"` or `"delete"`). `None` when the +/// bytes are not such a frame, or both fields are present. A put's `schemaId` is its document's. +pub fn missing_required_write_field(bytes: &[u8]) -> Option<(&'static str, &'static str)> { + let rmpv::Value::Map(entries) = rmpv::decode::read_value(&mut &*bytes).ok()? else { + return None; + }; + let field = |entries: &[(rmpv::Value, rmpv::Value)], name: &str| { + entries + .iter() + .find(|(key, _)| key.as_str() == Some(name)) + .map(|(_, value)| value.clone()) + .filter(|value| !value.is_nil()) + }; + match field(&entries, "schemaVersion")?.as_str()? { + "cultnet.document_put_raw.v0" => match field(&entries, "document") { + Some(rmpv::Value::Map(document)) => { + field(&document, "schemaId").is_none().then_some(("put", "schemaId")) + } + _ => Some(("put", "document")), + }, + "cultnet.document_delete.v0" => field(&entries, "schemaId").is_none().then_some(("delete", "schemaId")), + _ => None, + } +} diff --git a/packages/cultnet-rs/src/rudp.rs b/packages/cultnet-rs/src/rudp.rs index 6a2aa20b..eae7a2ee 100644 --- a/packages/cultnet-rs/src/rudp.rs +++ b/packages/cultnet-rs/src/rudp.rs @@ -468,6 +468,14 @@ impl CultNetRudpSession { self.create_disconnect(RUDP_REFUSED_PACKET_REASON.to_vec()) } + /// Ends a session because the server itself could not answer a write it owes an answer to (its answer + /// could not be encoded, queued or sent). The peer did nothing wrong: this goodbye is not the + /// "session refused a packet" of a protocol refusal. + pub fn end_local_failure_session(&mut self) -> CultNetRudpPacket { + self.reset_peer_state(); + self.create_disconnect(RUDP_LOCAL_FAILURE_REASON.to_vec()) + } + /// Ends a session that owes its peer a packet that can never be sent as /// built (see `is_permanent_send_error`). Resending it would fail forever, /// and dropping it would leave a reliable sequence the peer waits on for @@ -2609,6 +2617,8 @@ where /// Sent to a peer whose session refused one of its packets, and reported to /// the local caller as the disconnect reason. const RUDP_REFUSED_PACKET_REASON: &[u8] = b"session refused a packet"; +/// Goodbye reason of a session whose server could not answer a write (`end_local_failure_session`). +const RUDP_LOCAL_FAILURE_REASON: &[u8] = b"server could not answer a write"; const SESSION_TIMED_OUT_REASON: &[u8] = b"session timed out"; /// Prefix of the goodbye reason for a session that owed a packet which can /// never be sent; the failure's fixed name (`send_error_code`) follows it. diff --git a/packages/cultnet-rs/tests/write_receipt.rs b/packages/cultnet-rs/tests/write_receipt.rs new file mode 100644 index 00000000..6d06a68c --- /dev/null +++ b/packages/cultnet-rs/tests/write_receipt.rs @@ -0,0 +1,155 @@ +//! Wire parity with contracts/cultnet/interop/write-receipt-vectors.json: the content hash, and the two new frames. + +use cultnet_rs::{ + CultNetErrorCode, CultNetErrorDetails, CultNetMessage, CultNetWireContract, decode_cultnet_message_from_slice, + encode_cultnet_message_to_vec, write_content_hash, +}; + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +#[test] +fn the_content_hash_matches_the_spec_vectors() { + let a = [0x81, 0xA2, 0x69, 0x64, 0xA1, 0x31]; + let b = [0x81, 0xA2, 0x69, 0x64, 0xA1, 0x32]; + let cases = [ + (write_content_hash("put", "note.v0", "note:1", Some(&a)), "7051c10071123b6ade50fbd5075e3859cae4ea70ce358a24a0b0c6980090e95d"), + (write_content_hash("put", "note.v0", "note:1", Some(&b)), "58d3fc6b858d2605a68550c9d4811ccdca6b9db7a0c3bfebf304d279257f6f9f"), + (write_content_hash("put", "note.v0", "note:2", Some(&a)), "a248cd920e58d41bc6ccca9e108030b1d491239000047fef088122364a32b6f8"), + (write_content_hash("delete", "note.v0", "note:1", None), "232b0f96651b28aa8ab7673a6106944a9f6a8d66852b6e21291a23ed084b8b37"), + ]; + for (hash, expected) in cases { + assert_eq!(hex(&hash.unwrap()), expected); + } +} + +#[test] +fn the_new_frames_decode_back_equal() { + let admitted = CultNetMessage::WriteAdmitted { + message_id: "put-1".into(), + operation: "put".into(), + schema_id: "note.v0".into(), + record_key: "note:1".into(), + }; + let bytes = encode_cultnet_message_to_vec(&admitted, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert_eq!( + decode_cultnet_message_from_slice(&bytes, CultNetWireContract::CultNetSchemaV0).unwrap(), + admitted + ); + // error.v1 with a nil id round-trips as nil, and the code enum is v0's. + let error = CultNetMessage::ErrorV1 { + error: "message_invalid: unreadable frame".into(), + code: Some(CultNetErrorCode::MessageInvalid), + details: None, + message_id: None, + }; + let bytes = encode_cultnet_message_to_vec(&error, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert_eq!( + decode_cultnet_message_from_slice(&bytes, CultNetWireContract::CultNetSchemaV0).unwrap(), + error + ); +} + +fn unhex(text: &str) -> Vec { + (0..text.len()).step_by(2).map(|i| u8::from_str_radix(&text[i..i + 2], 16).unwrap()).collect() +} + +/// The spec's wire vectors (write-receipt-vectors.json `wire`): the bytes decode to the vector's fields, not +/// merely to something that encodes back to itself. +#[test] +fn the_spec_wire_vectors_decode_to_their_expected_fields() { + let admitted = decode_cultnet_message_from_slice( + &unhex("85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"), + CultNetWireContract::CultNetSchemaV0, + ) + .unwrap(); + assert_eq!( + admitted, + CultNetMessage::WriteAdmitted { + message_id: "put-1".into(), + operation: "put".into(), + schema_id: "note.v0".into(), + record_key: "note:1".into(), + } + ); + let refused = decode_cultnet_message_from_slice( + &unhex("86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D93677726974655F6E6F745F617574686F72697A65643A2074686520636174616C6F6720726566757365642074686520646F63756D656E74AB726F7574696E6748696E74C0A4636F6465B477726974655F6E6F745F617574686F72697A6564A764657461696C7384A56669656C64A97265636F72644B6579A576616C7565A66E6F74653A32A461734F66C0A763757272656E74C0A96D6573736167654964A57075742D32"), + CultNetWireContract::CultNetSchemaV0, + ) + .unwrap(); + let CultNetMessage::ErrorV1 { error, code, details, message_id } = refused else { + panic!("expected error.v1"); + }; + assert_eq!(error, "write_not_authorized: the catalog refused the document"); + assert_eq!(code, Some(CultNetErrorCode::WriteNotAuthorized)); + assert_eq!(message_id.as_deref(), Some("put-2")); + let details = details.unwrap(); + assert_eq!(details.field.as_deref(), Some("recordKey")); + assert_eq!(details.value.as_deref(), Some("note:2")); + assert_eq!((details.as_of, details.current), (None, None)); +} + +/// Spec vector `error_v1_limit_reached`: the bytes the server must send, and what they decode to. +#[test] +fn error_v1_limit_reached_is_the_spec_vector() { + let vector = "86AD736368656D6156657273696F6EB063756C746E65742E6572726F722E7631A56572726F72D93B6C696D69745F726561636865643A207468652073657373696F6E27732072656D656D62657265642D7772697465206C696D69742069732066756C6CAB726F7574696E6748696E74C0A4636F6465AD6C696D69745F72656163686564A764657461696C7384A56669656C64C0A576616C7565C0A461734F66C0A763757272656E74C0A96D6573736167654964A87075742D31303235"; + let message = CultNetMessage::ErrorV1 { + error: "limit_reached: the session's remembered-write limit is full".into(), + code: Some(CultNetErrorCode::LimitReached), + details: Some(CultNetErrorDetails::default()), + message_id: Some("put-1025".into()), + }; + let bytes = encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert_eq!(bytes, unhex(vector), "encoded bytes must equal the spec vector"); + let CultNetMessage::ErrorV1 { error, code, details, message_id } = + decode_cultnet_message_from_slice(&unhex(vector), CultNetWireContract::CultNetSchemaV0).unwrap() + else { + panic!("expected error.v1"); + }; + assert_eq!(error, "limit_reached: the session's remembered-write limit is full"); + assert_eq!(code, Some(CultNetErrorCode::LimitReached)); + assert_eq!(message_id.as_deref(), Some("put-1025")); + assert_eq!(details, Some(CultNetErrorDetails::default())); +} + +/// Spec vectors `write_admitted_put` and `write_admitted_delete`: the encoder emits the spec's field order. +#[test] +fn write_admitted_encodes_to_the_spec_vector_bytes() { + let cases = [ + ("put-1", "put", "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A57075742D31A96F7065726174696F6EA3707574A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"), + ("del-1", "delete", "85AD736368656D6156657273696F6EB963756C746E65742E77726974655F61646D69747465642E7630A96D6573736167654964A564656C2D31A96F7065726174696F6EA664656C657465A8736368656D614964A76E6F74652E7630A97265636F72644B6579A66E6F74653A31"), + ]; + for (id, operation, vector) in cases { + let message = CultNetMessage::WriteAdmitted { + message_id: id.into(), + operation: operation.into(), + schema_id: "note.v0".into(), + record_key: "note:1".into(), + }; + let bytes = encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).unwrap(); + assert_eq!(bytes, unhex(vector)); + } +} + +/// `limit_reached` is in the error.v1 enum only: a v0 error neither encodes nor decodes it. +#[test] +fn error_v0_refuses_the_v1_only_limit_reached_code() { + let v0 = CultNetMessage::Error { + error: "x".into(), + code: Some(CultNetErrorCode::LimitReached), + details: None, + }; + assert!(encode_cultnet_message_to_vec(&v0, CultNetWireContract::CultNetSchemaV0).is_err()); + let v1 = CultNetMessage::ErrorV1 { + error: "x".into(), + code: Some(CultNetErrorCode::LimitReached), + details: None, + message_id: None, + }; + let v1_bytes = encode_cultnet_message_to_vec(&v1, CultNetWireContract::CultNetSchemaV0).unwrap(); + let at = v1_bytes.windows(16).position(|w| w == b"cultnet.error.v1").unwrap(); + let mut as_v0 = v1_bytes.clone(); + as_v0[at + 15] = b'0'; + assert!(decode_cultnet_message_from_slice(&as_v0, CultNetWireContract::CultNetSchemaV0).is_err()); +}