cultmesh-rs: a refusal carries fixed text, never a sink's or source's error
The refusal sent to a rejected peer carried the rejection's Display text, which for SinkRefused and SnapshotSourceFailed is the sink's or source's own error. Odin's sink formats the peer-supplied document type into its error, so a peer's input came back on the wire. The refusal now carries fixed text per rejection variant (REFUSAL_TEXTS). The rejection returned from poll_once still carries the detail for the caller; the server itself logs nothing. publish_cultnet_message_to_rudp_catalog no longer quotes a catalog's text unfiltered: it reports "refused the put", quoting the text only when it is one of the fixed refusals. The test fixtures' sink and source errors carry a canary; the refused peer's text is the fixed one and never contains it. A fake catalog that refuses with a canary of its own shows the publisher's error stays fixed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
ead1a25aff
commit
b3176de1f8
3 changed files with 165 additions and 22 deletions
|
|
@ -867,7 +867,8 @@ fn unix_millis() -> u128 {
|
|||
///
|
||||
/// Each call owns a fresh connection epoch and returns only after the catalog
|
||||
/// acknowledges reliable application admission. A catalog that refuses the
|
||||
/// message answers with a `cultnet.error.v0`, returned as an error naming it.
|
||||
/// message answers with a `cultnet.error.v0`, returned as an error that names
|
||||
/// the refusal when it is one of the fixed texts a CultMesh catalog sends.
|
||||
pub fn publish_cultnet_message_to_rudp_catalog(
|
||||
message: &cultnet_rs::CultNetMessage,
|
||||
options: CultMeshRudpDocumentPublishOptions,
|
||||
|
|
@ -897,10 +898,15 @@ pub fn publish_cultnet_message_to_rudp_catalog(
|
|||
)
|
||||
{
|
||||
let _ = client.disconnect(b"document-refused".to_vec());
|
||||
anyhow::bail!(
|
||||
"CultMesh RUDP catalog {} refused the message: {error}",
|
||||
options.target
|
||||
);
|
||||
// The catalog's text is quoted only when it is one of the
|
||||
// fixed refusals: any other text could carry anything.
|
||||
if REFUSAL_TEXTS.contains(&error.as_str()) {
|
||||
anyhow::bail!(
|
||||
"CultMesh RUDP catalog {} refused the put: {error}",
|
||||
options.target
|
||||
);
|
||||
}
|
||||
anyhow::bail!("CultMesh RUDP catalog {} refused the put", options.target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1311,7 +1317,7 @@ mod tests {
|
|||
let mut server = CultMeshRudpDocumentServer::new(
|
||||
socket,
|
||||
|_receipt: CultMeshRudpRawDocumentReceipt| {
|
||||
anyhow::bail!("injected durable sink failure")
|
||||
anyhow::bail!("injected durable sink failure CANARY-7f3a")
|
||||
},
|
||||
|_query: &CultMeshRudpSnapshotQuery| Ok(Vec::new()),
|
||||
CultMeshSystemClock::default(),
|
||||
|
|
@ -1349,13 +1355,112 @@ mod tests {
|
|||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("refused the message: injected durable sink failure"),
|
||||
.ends_with("refused the put: the catalog refused the document"),
|
||||
"{error:#}"
|
||||
);
|
||||
assert!(!error.to_string().contains("CANARY"), "{error:#}");
|
||||
assert!(waited < Duration::from_secs(5), "waited {waited:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A catalog's refusal text is quoted only when it is one of the fixed
|
||||
/// refusals a CultMesh catalog sends. Any other text, which could carry
|
||||
/// anything, stays out of the publisher's error.
|
||||
#[test]
|
||||
fn a_publisher_never_quotes_a_refusal_it_does_not_recognise() -> Result<()> {
|
||||
use cultnet_rs::{
|
||||
CultNetRudpPacketType, CultNetRudpSendOptions, CultNetRudpSession,
|
||||
CultNetRudpSessionOptions, decode_rudp_packet, encode_rudp_packet,
|
||||
};
|
||||
let temp = tempfile::tempdir()?;
|
||||
let node = CultMesh::create_node(
|
||||
temp.path().join("cultmesh.cc"),
|
||||
TestDocuments,
|
||||
Default::default(),
|
||||
)?;
|
||||
let socket = UdpSocket::bind("127.0.0.1:0")?;
|
||||
socket.set_read_timeout(Some(Duration::from_millis(5)))?;
|
||||
let target = socket.local_addr()?;
|
||||
let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let catalog_stop = stop.clone();
|
||||
// A catalog that answers any put with a refusal of its own wording.
|
||||
let catalog = thread::spawn(move || -> Result<()> {
|
||||
let mut session: Option<CultNetRudpSession> = None;
|
||||
let mut wire = vec![0_u8; 65_535];
|
||||
while !catalog_stop.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
let Ok((bytes, peer)) = socket.recv_from(&mut wire) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(packet) = decode_rudp_packet(&wire[..bytes]) else {
|
||||
continue;
|
||||
};
|
||||
if packet.packet_type == CultNetRudpPacketType::Connect {
|
||||
let mut accepted = CultNetRudpSession::new(CultNetRudpSessionOptions {
|
||||
connection_id: packet.connection_id,
|
||||
initial_sequence: None,
|
||||
resend_delay_ms: 10_000,
|
||||
max_pending_reliable_packets: Some(64),
|
||||
});
|
||||
let accept = accepted.accept_connect(&packet, 0, Vec::new())?;
|
||||
socket.send_to(&encode_rudp_packet(&accept)?, peer)?;
|
||||
session = Some(accepted);
|
||||
continue;
|
||||
}
|
||||
let Some(session) = session.as_mut() else {
|
||||
continue;
|
||||
};
|
||||
let Ok(received) = session.receive(&packet, 1) else {
|
||||
continue;
|
||||
};
|
||||
for _put in received.delivered {
|
||||
let refusal = encode_cultnet_message_to_vec(
|
||||
&cultnet_rs::CultNetMessage::Error {
|
||||
error: "no such tenant: CANARY-7f3a".into(),
|
||||
code: None,
|
||||
details: None,
|
||||
},
|
||||
CultNetWireContract::CultNetSchemaV0,
|
||||
)?;
|
||||
let options = CultNetRudpSendOptions {
|
||||
reliable: false,
|
||||
ordered: false,
|
||||
sequenced: false,
|
||||
now_ms: 2,
|
||||
reliable_expire_after_ms: None,
|
||||
};
|
||||
for packet in session.send_many("schema", refusal, options, Some(1200))? {
|
||||
socket.send_to(&encode_rudp_packet(&packet)?, peer)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
|
||||
let error = node
|
||||
.publish_document_to_rudp_catalog(
|
||||
"note",
|
||||
&Note {
|
||||
body: "refused by a stranger".into(),
|
||||
},
|
||||
CultMeshRudpDocumentPublishOptions {
|
||||
target,
|
||||
connect_timeout: Duration::from_secs(2),
|
||||
flush_timeout: Duration::from_secs(10),
|
||||
poll_interval: Duration::from_millis(2),
|
||||
resend_delay_ms: 5,
|
||||
..CultMeshRudpDocumentPublishOptions::default()
|
||||
},
|
||||
)
|
||||
.expect_err("a refused document must fail closed");
|
||||
stop.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
catalog.join().expect("catalog thread should not panic")?;
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
format!("CultMesh RUDP catalog {target} refused the put")
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_snapshot_preserves_caller_challenge_and_raw_response() -> Result<()> {
|
||||
let expected = cultnet_rs::CultNetRawDocumentRecord {
|
||||
|
|
|
|||
|
|
@ -274,6 +274,36 @@ impl std::fmt::Display for CultMeshRudpRejectionReason {
|
|||
}
|
||||
}
|
||||
|
||||
/// The texts a refused peer is sent, one per rejection variant. They are fixed:
|
||||
/// a sink's or source's error text can quote what a peer sent, so it stays with
|
||||
/// the caller that receives the rejection and never reaches the wire.
|
||||
pub(crate) const REFUSAL_TEXTS: [&str; 8] = [
|
||||
"the catalog refused the document",
|
||||
"the document can never be served",
|
||||
"the snapshot source failed",
|
||||
"the snapshot has too many documents",
|
||||
"the snapshot response is too large",
|
||||
"the snapshot response could not be encoded",
|
||||
"the retained payload budget is full",
|
||||
"the snapshot response could not be queued",
|
||||
];
|
||||
|
||||
impl CultMeshRudpRejectionReason {
|
||||
/// The fixed text the refused peer is sent for this variant.
|
||||
fn refusal_text(&self) -> &'static str {
|
||||
REFUSAL_TEXTS[match self {
|
||||
Self::SinkRefused(_) => 0,
|
||||
Self::DocumentUnservable { .. } => 1,
|
||||
Self::SnapshotSourceFailed(_) => 2,
|
||||
Self::SnapshotTooManyDocuments { .. } => 3,
|
||||
Self::SnapshotResponseTooLarge { .. } => 4,
|
||||
Self::ResponseEncodingFailed(_) => 5,
|
||||
Self::PayloadBudgetFull => 6,
|
||||
Self::ResponseQueueFailed(_) => 7,
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct CultMeshRudpApplicationRejection {
|
||||
pub session: CultMeshRudpSessionKey,
|
||||
|
|
@ -359,9 +389,11 @@ where
|
|||
///
|
||||
/// Application rejection invalidates only the responsible peer session and
|
||||
/// is returned as data so a daemon can log it and keep serving. The peer is
|
||||
/// sent the refusal as a `cultnet.error.v0` carrying the reason's text, then
|
||||
/// a goodbye; neither acknowledges the refused message. Local socket and
|
||||
/// server-state failures remain errors.
|
||||
/// sent the refusal as a `cultnet.error.v0` carrying fixed text for the
|
||||
/// rejection's variant, then a goodbye; neither acknowledges the refused
|
||||
/// message. The rejection's own detail, which can quote a sink's or
|
||||
/// source's error, stays with the caller. Local socket and server-state
|
||||
/// failures remain errors.
|
||||
pub fn poll_once(&mut self) -> Result<CultMeshRudpPollOutcome> {
|
||||
self.maintain()?;
|
||||
let mut wire = vec![0_u8; MAX_UDP_DATAGRAM_BYTES];
|
||||
|
|
@ -460,7 +492,7 @@ where
|
|||
now_unix,
|
||||
now_monotonic,
|
||||
)? {
|
||||
self.end_rejected_session(key, rejection.reason.to_string())?;
|
||||
self.end_rejected_session(key, rejection.reason.refusal_text())?;
|
||||
return Ok(CultMeshRudpPollOutcome::ApplicationRejected(rejection));
|
||||
}
|
||||
}
|
||||
|
|
@ -757,12 +789,12 @@ where
|
|||
/// has emptied: it must not acknowledge the refused message, because a
|
||||
/// publisher reads that acknowledgement as admission. A refusal that cannot
|
||||
/// be encoded or queued is not sent; the goodbye still is.
|
||||
fn end_rejected_session(&mut self, key: CultMeshRudpSessionKey, reason: String) -> Result<()> {
|
||||
fn end_rejected_session(&mut self, key: CultMeshRudpSessionKey, reason: &str) -> Result<()> {
|
||||
let Some(mut entry) = self.sessions.remove(&key) else {
|
||||
return Ok(());
|
||||
};
|
||||
let refusal = CultNetMessage::Error {
|
||||
error: reason,
|
||||
error: reason.into(),
|
||||
code: None,
|
||||
details: None,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ impl CultMeshRudpRawDocumentSink for Sink {
|
|||
let mut state = self.0.lock().unwrap();
|
||||
if state.failures_remaining > 0 {
|
||||
state.failures_remaining -= 1;
|
||||
anyhow::bail!("injected sink failure");
|
||||
anyhow::bail!("injected sink failure CANARY-7f3a");
|
||||
}
|
||||
state.receipts.push(receipt);
|
||||
Ok(())
|
||||
|
|
@ -95,7 +95,7 @@ impl CultMeshRudpSnapshotSource for Source {
|
|||
state.calls += 1;
|
||||
if state.failures_remaining > 0 {
|
||||
state.failures_remaining -= 1;
|
||||
anyhow::bail!("injected source failure");
|
||||
anyhow::bail!("injected source failure CANARY-7f3a");
|
||||
}
|
||||
Ok(state.documents.clone())
|
||||
}
|
||||
|
|
@ -335,7 +335,8 @@ fn poll_until_rejected_or_stored<Q: CultMeshRudpSnapshotSource>(
|
|||
anyhow::bail!("the put was neither refused nor stored")
|
||||
}
|
||||
|
||||
/// What a refused peer is sent: a `cultnet.error.v0`, then the goodbye. The
|
||||
/// What a refused peer is sent: a `cultnet.error.v0` carrying fixed text, never
|
||||
/// a sink's or source's error (the fixtures' errors carry a canary), then the goodbye. The
|
||||
/// refused message is never acknowledged, so its receipt stays pending until the
|
||||
/// goodbye invalidates it. Returns the refusal's text.
|
||||
fn refusal_seen_by(
|
||||
|
|
@ -362,6 +363,7 @@ fn refusal_seen_by(
|
|||
panic!("the refused peer was sent no refusal: {refusal:?}");
|
||||
};
|
||||
assert_eq!((code, details), (None, None));
|
||||
assert!(!error.contains("CANARY"), "a sink's or source's text reached the peer: {error}");
|
||||
assert_eq!(
|
||||
client.reliable_send_status(receipt),
|
||||
CultNetRudpReliableSendStatus::Pending,
|
||||
|
|
@ -662,12 +664,13 @@ fn application_rejection_is_nonfatal_peer_scoped_refused_to_the_peer_and_unackno
|
|||
assert_eq!(rejection.message_id, "rejected-put");
|
||||
assert_eq!(
|
||||
rejection.reason,
|
||||
CultMeshRudpRejectionReason::SinkRefused("injected sink failure".into())
|
||||
CultMeshRudpRejectionReason::SinkRefused("injected sink failure CANARY-7f3a".into())
|
||||
);
|
||||
assert_eq!(server.session_count(), 2);
|
||||
// The caller keeps the sink's text; the peer is sent fixed text, never it.
|
||||
assert_eq!(
|
||||
refusal_seen_by(&mut publisher, &publish_receipt)?,
|
||||
"injected sink failure"
|
||||
"the catalog refused the document"
|
||||
);
|
||||
drain_until_idle(&mut server)?;
|
||||
|
||||
|
|
@ -690,12 +693,12 @@ fn application_rejection_is_nonfatal_peer_scoped_refused_to_the_peer_and_unackno
|
|||
assert_eq!(rejection.message_id, "rejected-snapshot");
|
||||
assert_eq!(
|
||||
rejection.reason,
|
||||
CultMeshRudpRejectionReason::SnapshotSourceFailed("injected source failure".into())
|
||||
CultMeshRudpRejectionReason::SnapshotSourceFailed("injected source failure CANARY-7f3a".into())
|
||||
);
|
||||
assert_eq!(server.session_count(), 1);
|
||||
assert_eq!(
|
||||
refusal_seen_by(&mut snapshot_client, &snapshot_receipt)?,
|
||||
"injected source failure"
|
||||
"the snapshot source failed"
|
||||
);
|
||||
drain_until_idle(&mut server)?;
|
||||
|
||||
|
|
@ -842,7 +845,7 @@ fn payload_and_snapshot_output_budgets_fail_closed() -> Result<()> {
|
|||
assert_eq!(snapshot_server.session_count(), 0);
|
||||
assert_eq!(
|
||||
refusal_seen_by(&mut snapshot_client, &receipt)?,
|
||||
rejection.reason.to_string()
|
||||
"the snapshot response is too large"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -904,7 +907,10 @@ fn a_put_the_server_could_never_serve_is_refused_and_one_at_the_bound_is_served(
|
|||
assert_eq!(server.session_count(), 1);
|
||||
// The peer hears the refusal and why, rather than waiting on a put that
|
||||
// will never be acknowledged.
|
||||
assert_eq!(refusal_seen_by(&mut refused, &refused_receipt)?, sentence);
|
||||
assert_eq!(
|
||||
refusal_seen_by(&mut refused, &refused_receipt)?,
|
||||
"the document can never be served"
|
||||
);
|
||||
|
||||
send(
|
||||
&mut writer,
|
||||
|
|
|
|||
Loading…
Reference in a new issue