diff --git a/Cargo.lock b/Cargo.lock index e792fdd..ad75baa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1167,6 +1167,7 @@ dependencies = [ "sha2", "syn 2.0.119", "tempfile", + "unicode-general-category", "ureq", "uuid", ] diff --git a/crates/mind-body/Cargo.toml b/crates/mind-body/Cargo.toml index ffacef1..e21212b 100644 --- a/crates/mind-body/Cargo.toml +++ b/crates/mind-body/Cargo.toml @@ -46,6 +46,7 @@ rmp-serde = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" +unicode-general-category = "1" ureq = { version = "~3.3", default-features = false } uuid = { version = "1", default-features = false, features = ["v5"] } diff --git a/crates/mind-body/src/address/tests.rs b/crates/mind-body/src/address/tests.rs index 79ae86b..23e2827 100644 --- a/crates/mind-body/src/address/tests.rs +++ b/crates/mind-body/src/address/tests.rs @@ -13,6 +13,7 @@ fn said(text: &str, mentions: &[&str]) -> ChatDelivery { image: None, basis: None, form: Form::Message, + reply_to: crate::chat_client::ReplyLink::Absent, sender: "@op:chat".into(), sender_display: "Op".into(), at_unix_ms: 0, diff --git a/crates/mind-body/src/chat_client.rs b/crates/mind-body/src/chat_client.rs index a5cbe07..8816cc7 100644 --- a/crates/mind-body/src/chat_client.rs +++ b/crates/mind-body/src/chat_client.rs @@ -21,6 +21,10 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; pub const PENDING_REQUEST_V1: &str = "gamecult.persona.chat_pending_request.v1"; /// The pending request, the same tuple. Asking by this kind is asking for the answer with the addressing facts. pub const PENDING_REQUEST: &str = "gamecult.persona.chat_pending_request.v2"; +/// The pending request that asks for the reply link, the same tuple. Opt-in: only a client with the link on sends it. +pub const PENDING_REQUEST_V4: &str = "gamecult.persona.chat_pending_request.v3"; +/// The answer a flag-on mouth gives it: a delivery with `reply_to` at tuple index 14. Never asked for by default. +pub const PENDING_V4: &str = "gamecult.persona.chat_pending.v4"; /// The answer to it: a delivery with its `basis` and `form` at tuple indices 12 and 13. pub const PENDING: &str = "gamecult.persona.chat_pending.v3"; /// The answer a mouth gives a v1 request: no addressing facts, decoded as `basis: None, form: Message`. @@ -71,6 +75,229 @@ pub struct ChatDelivery { /// What kind of event it is: tuple index 13. An answer of an older mouth ends before it and reads as a message. #[serde(default)] pub form: Form, + /// The earlier event this message replies to, as a `chat_pending.v4` mouth quotes it: tuple index 14. An answer of an older mouth + /// ends before it and reads as absent, and so does an explicit nil. Display-only evidence: no addressing rule reads it. + #[serde(default, skip_serializing_if = "ReplyLink::sends_nothing")] + pub reply_to: ReplyLink, +} + +/// The longest excerpt the mouth quotes, in Unicode code points (its `limits.replyExcerpt`). +pub const REPLY_EXCERPT_CHARS: usize = 240; + +/// The quote of the event a message replies to: `[event_id, sender, excerpt]`. `excerpt` is untrusted text, none when the mouth sent +/// none (redacted, edited, or not text). Built only by `ReplyLink`'s decode, which holds it to the mouth's contract. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ReplyTo { + pub event_id: String, + pub sender: String, + pub excerpt: Option, +} + +/// How a reply link out of the mouth's contract fails it, first by tuple order. A class and nothing the mouth sent, so no path can echo +/// what was read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReplyViolation { + /// Not an array, or an array of arity other than 3. + NotATriple, + /// An event id that is not a string, is empty, or is not the shape of one. + BadEventId, + /// A sender that is not a string, is empty, or is not the shape of a Matrix user id. + BadSenderId, + /// An excerpt that is neither a string nor nil. + ExcerptNotText, + /// An excerpt over `REPLY_EXCERPT_CHARS` code points as sent, before invisible characters are stripped. + ExcerptTooLong, +} + +impl ReplyViolation { + /// The log token of the class. + pub fn name(self) -> &'static str { + match self { + Self::NotATriple => "not_a_triple", + Self::BadEventId => "bad_event_id", + Self::BadSenderId => "bad_sender_id", + Self::ExcerptNotText => "excerpt_not_text", + Self::ExcerptTooLong => "excerpt_too_long", + } + } +} + +/// Whether `text` has the shape of a Matrix user id: `@localpart:server`, at most 255 bytes. A shape check, no directory lookup. The +/// localpart is any ASCII graphic byte (historical and appservice ids such as `@_discord_123:host` are legal); the server name is a +/// hostname, IPv4 or `[IPv6]` with an optional port. +fn matrix_user_id(text: &str) -> bool { + let Some((localpart, server)) = text.strip_prefix('@').and_then(|rest| rest.split_once(':')) else { return false }; + text.len() <= 255 + && !localpart.is_empty() + && localpart.bytes().all(|byte| byte.is_ascii_graphic()) + && !server.is_empty() + && server.bytes().all(|byte| byte.is_ascii_alphanumeric() || b".:_[]-".contains(&byte)) +} + +/// Whether `text` has the shape of an event id: `$` and at least one ASCII graphic byte, at most 255 bytes. Deliberately not a base64 +/// check: room versions 3 to 12 and the legacy `$opaque:server` ids all pass, because the mouth forwards whatever the homeserver named. +fn event_id_shape(text: &str) -> bool { + text.len() <= 255 && text.strip_prefix('$').is_some_and(|rest| !rest.is_empty() && rest.bytes().all(|byte| byte.is_ascii_graphic())) +} + +/// The excerpt without the characters that show nothing: variation selectors, the tag block, and every format or private-use character. +/// Removal only; nothing is trimmed, replaced or reordered. +fn without_invisible(text: &str) -> String { + use unicode_general_category::{GeneralCategory, get_general_category}; + text.chars() + .filter(|&c| { + !matches!(c, '\u{FE00}'..='\u{FE0F}' | '\u{E0100}'..='\u{E01EF}' | '\u{E0000}'..='\u{E007F}') + && !matches!(get_general_category(c), GeneralCategory::Format | GeneralCategory::PrivateUse) + }) + .collect() +} + +/// A delivery's reply link, tuple index 14: absent, a quote within the mouth's contract, or dropped whole for leaving it. A defective link +/// never fails the poll and is never repaired or clipped; `reply_warnings` names it, and only `quote` reaches a model. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum ReplyLink { + #[default] + Absent, + Quote(ReplyTo), + Dropped(ReplyViolation), +} + +impl ReplyLink { + /// The quote, only when the link was within the contract. + pub fn quote(&self) -> Option<&ReplyTo> { + match self { + Self::Quote(quote) => Some(quote), + _ => None, + } + } + + fn sends_nothing(&self) -> bool { + !matches!(self, Self::Quote(_)) + } +} + +impl Serialize for ReplyLink { + fn serialize(&self, serializer: S) -> Result { + match self { + Self::Quote(quote) => quote.serialize(serializer), + _ => serializer.serialize_none(), + } + } +} + +impl<'de> Deserialize<'de> for ReplyLink { + fn deserialize>(deserializer: D) -> Result { + // Whatever one element of the document holds, read whole so it can be judged after: an element that errors mid-read cannot be resumed. + enum Element { + Nil, + Text(String), + List(Vec), + Other, + } + + struct ElementVisitor; + + impl<'de> serde::de::Visitor<'de> for ElementVisitor { + type Value = Element; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a reply link element") + } + + fn visit_unit(self) -> Result { + Ok(Element::Nil) + } + + fn visit_none(self) -> Result { + Ok(Element::Nil) + } + + fn visit_str(self, text: &str) -> Result { + Ok(Element::Text(text.to_string())) + } + + fn visit_string(self, text: String) -> Result { + Ok(Element::Text(text)) + } + + fn visit_seq>(self, mut seq: A) -> Result { + let mut items = Vec::new(); + while let Some(item) = seq.next_element::()? { + items.push(item); + } + Ok(Element::List(items)) + } + + fn visit_map>(self, mut map: A) -> Result { + while map.next_entry::()?.is_some() {} + Ok(Element::Other) + } + + fn visit_bool(self, _: bool) -> Result { + Ok(Element::Other) + } + + fn visit_i64(self, _: i64) -> Result { + Ok(Element::Other) + } + + fn visit_u64(self, _: u64) -> Result { + Ok(Element::Other) + } + + fn visit_f64(self, _: f64) -> Result { + Ok(Element::Other) + } + + fn visit_bytes(self, _: &[u8]) -> Result { + Ok(Element::Other) + } + } + + impl<'de> Deserialize<'de> for Element { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_any(ElementVisitor) + } + } + + Ok(match Element::deserialize(deserializer)? { + Element::Nil => Self::Absent, + Element::List(items) if items.len() == 3 => { + let mut items = items.into_iter(); + let (event_id, sender, excerpt) = (items.next(), items.next(), items.next()); + match (event_id, sender, excerpt) { + (Some(Element::Text(event_id)), _, _) if !event_id_shape(&event_id) => Self::Dropped(ReplyViolation::BadEventId), + (Some(Element::Text(_)), Some(Element::Text(sender)), _) if !matrix_user_id(&sender) => Self::Dropped(ReplyViolation::BadSenderId), + (Some(Element::Text(event_id)), Some(Element::Text(sender)), Some(excerpt)) => match excerpt { + Element::Nil => Self::Quote(ReplyTo { event_id, sender, excerpt: None }), + Element::Text(text) if text.chars().take(REPLY_EXCERPT_CHARS + 1).count() > REPLY_EXCERPT_CHARS => Self::Dropped(ReplyViolation::ExcerptTooLong), + Element::Text(text) => { + let shown = without_invisible(&text); + Self::Quote(ReplyTo { event_id, sender, excerpt: if shown.is_empty() { None } else { Some(shown) } }) + } + _ => Self::Dropped(ReplyViolation::ExcerptNotText), + }, + (Some(Element::Text(_)), _, _) => Self::Dropped(ReplyViolation::BadSenderId), + _ => Self::Dropped(ReplyViolation::BadEventId), + } + } + _ => Self::Dropped(ReplyViolation::NotATriple), + }) + } +} + +/// The log line for each delivery whose reply link was dropped and whose event id is not yet in `warned`, which it grows: one warning per +/// event however often the answer is polled. The event id is escaped (`{:?}`) so it cannot start another line, and the line holds nothing +/// else the mouth sent. +pub fn reply_warnings(pending: &ChatPending, warned: &mut std::collections::BTreeSet) -> Vec { + pending + .deliveries + .iter() + .filter_map(|delivery| match &delivery.reply_to { + ReplyLink::Dropped(violation) if warned.insert(delivery.event_id.clone()) => Some(format!("reply_to dropped event={:?} violation={}", delivery.event_id, violation.name())), + _ => None, + }) + .collect() } /// Why the mouth found a delivery to be for the Persona, by what Matrix says (Bifrost `docs/bridge.md` "Addressing facts"). `Tag` and diff --git a/crates/mind-body/src/chat_client/tests.rs b/crates/mind-body/src/chat_client/tests.rs index 828c68c..db3b7b7 100644 --- a/crates/mind-body/src/chat_client/tests.rs +++ b/crates/mind-body/src/chat_client/tests.rs @@ -21,6 +21,11 @@ const PENDING_UNPROVEN_NOTICE_TAILED_BYTES: &[u8] = include_bytes!("../../tests/ const PENDING_TAIL_BYTES: &[u8] = include_bytes!("../../tests/fixtures/persona-chat-v6/gamecult.persona.chat_pending.v3.tail.msgpack"); // The four-field `chat_pending.v3` with `tail_gaps`: Bifrost b2e0bb2's own golden bytes, copied unchanged. const PENDING_GAPS_BYTES: &[u8] = include_bytes!("../../tests/fixtures/persona-chat-v7/gamecult.persona.chat_pending.v3.tail-gaps.msgpack"); +// Bifrost dc35c4a2's own golden bytes for the reply link (`tests/fixtures/persona-chat-v6/` there), copied unchanged as `persona-chat-v8/`. +const PENDING_V4_REPLY_BYTES: &[u8] = include_bytes!("../../tests/fixtures/persona-chat-v8/gamecult.persona.chat_pending.v4.reply-to.msgpack"); +const PENDING_V4_REDACTED_BYTES: &[u8] = include_bytes!("../../tests/fixtures/persona-chat-v8/gamecult.persona.chat_pending.v4.reply-to-redacted.msgpack"); +const PENDING_V4_NONE_BYTES: &[u8] = include_bytes!("../../tests/fixtures/persona-chat-v8/gamecult.persona.chat_pending.v4.no-reply-to.msgpack"); +const PENDING_REQUEST_V3_BYTES: &[u8] = include_bytes!("../../tests/fixtures/persona-chat-v8/gamecult.persona.chat_pending_request.v3.msgpack"); const SENT_BYTES: &[u8] = include_bytes!("../../tests/fixtures/persona-chat-v2/gamecult.persona.chat_send_receipt.v2.sent.msgpack"); const ROOM_MOVED_BYTES: &[u8] = include_bytes!("../../tests/fixtures/persona-chat-v2/gamecult.persona.chat_send_receipt.v2.room-moved.msgpack"); const NIL_BASIS_BYTES: &[u8] = include_bytes!("../../tests/fixtures/persona-chat-v2/gamecult.persona.chat_send_request.v2.nil-basis.msgpack"); @@ -62,6 +67,7 @@ fn delivery(image: Option) -> ChatDelivery { image, basis: None, form: Form::Message, + reply_to: ReplyLink::Absent, } } @@ -241,3 +247,267 @@ fn the_pending_answer_decodes_tail_gaps_and_defaults_them_to_none() { empty_fourth.as_array_mut().unwrap().push(serde_json::json!([])); assert_eq!(decode::(&rmp_serde::to_vec(&empty_fourth).unwrap()).unwrap(), tailed, "an empty fourth element is the same answer"); } + +fn quote(sender: &str, excerpt: Option<&str>) -> ReplyLink { + ReplyLink::Quote(ReplyTo { event_id: "$q1".into(), sender: sender.into(), excerpt: excerpt.map(Into::into) }) +} + +/// The v4 answers carry `reply_to` at delivery index 14. The mouth always sends the fourth, empty `tail_gaps` element on a v4 answer, +/// which this side omits when empty, so a re-encode is the same answer without that trailing empty list. +#[test] +fn the_v4_golden_bytes_decode_and_the_reply_to_ones_reencode() { + let one = |reply_to| { + let mut delivery = delivery(None); + (delivery.is_direct, delivery.mentions, delivery.reply_to) = (false, vec!["@eureka:gamecult.org".into(), "@nova:gamecult.org".into()], reply_to); + ChatPending { persona: "eureka".into(), deliveries: vec![delivery], tail: Some(vec![]), tail_gaps: vec![] } + }; + let without_empty_gaps = |bytes: &[u8]| { + let mut value: serde_json::Value = rmp_serde::from_slice(bytes).unwrap(); + assert_eq!(value.as_array_mut().unwrap().pop(), Some(serde_json::json!([]))); + value + }; + for (bytes, expected) in [(PENDING_V4_REPLY_BYTES, one(quote("@alice:gamecult.org", Some("can someone look at the build?")))), (PENDING_V4_REDACTED_BYTES, one(quote("@alice:gamecult.org", None)))] { + assert_eq!(decode::(bytes).unwrap(), expected); + let reencoded: serde_json::Value = rmp_serde::from_slice(&encode(&expected).unwrap()).unwrap(); + assert_eq!(reencoded, without_empty_gaps(bytes)); + assert_eq!(reencoded[1][0].as_array().unwrap().len(), 15); + } + // An explicit nil at index 14 reads as none; a re-encode drops it, so this fixture is decode-only. + let none = decode::(PENDING_V4_NONE_BYTES).unwrap(); + assert_eq!(none, one(ReplyLink::Absent)); + assert_eq!(rmp_serde::from_slice::(&encode(&none).unwrap()).unwrap()[1][0].as_array().unwrap().len(), 14); +} + +#[test] +fn the_v1_to_v3_answers_still_decode_with_no_reply_to() { + for old in [PENDING_V1_BYTES, PENDING_NO_IMAGE_BYTES, PENDING_IMAGE_BYTES, PENDING_TAG_BYTES, PENDING_UNPROVEN_NOTICE_BYTES, PENDING_TAG_TAILED_BYTES, PENDING_TAIL_BYTES, PENDING_GAPS_BYTES] { + assert!(decode::(old).unwrap().deliveries.iter().all(|delivery| delivery.reply_to == ReplyLink::Absent)); + } + golden(PENDING_TAG_BYTES, &decode::(PENDING_TAG_BYTES).unwrap()); + golden(PENDING_GAPS_BYTES, &decode::(PENDING_GAPS_BYTES).unwrap()); +} + +/// The valid reply-to answer with its delivery's index 14 swapped for `element`, as msgpack. +fn answer_with(element: serde_json::Value) -> Vec { + let mut value: serde_json::Value = rmp_serde::from_slice(PENDING_V4_REPLY_BYTES).unwrap(); + value[1][0][14] = element; + rmp_serde::to_vec(&value).unwrap() +} + +fn link_of(bytes: &[u8]) -> ReplyLink { + decode::(bytes).unwrap().deliveries.remove(0).reply_to +} + +#[test] +fn a_bad_reply_to_drops_that_quote_and_the_rest_of_the_poll_decodes() { + let good: serde_json::Value = rmp_serde::from_slice(PENDING_V4_REPLY_BYTES).unwrap(); + let valid = good[1][0].clone(); + let mut bad = valid.clone(); + bad[14] = serde_json::json!(["$q1", "@alice:gamecult.org", 5]); + bad[3] = "$bad".into(); + let mut document = good.clone(); + document[1] = serde_json::json!([valid.clone(), bad, valid]); + let pending: ChatPending = decode(&rmp_serde::to_vec(&document).unwrap()).unwrap(); + let expected: ChatPending = decode(PENDING_V4_REPLY_BYTES).unwrap(); + assert_eq!(pending.deliveries.len(), 3); + assert_eq!((&pending.deliveries[0], &pending.deliveries[2]), (&expected.deliveries[0], &expected.deliveries[0])); + let dropped = &pending.deliveries[1]; + assert_eq!(dropped.reply_to, ReplyLink::Dropped(ReplyViolation::ExcerptNotText)); + assert!(dropped.reply_to.quote().is_none()); + assert_eq!(ChatDelivery { event_id: "$e1".into(), reply_to: expected.deliveries[0].reply_to.clone(), ..dropped.clone() }, expected.deliveries[0], "the delivery is otherwise unchanged"); + assert_eq!((&pending.persona, &pending.tail, &pending.tail_gaps), (&expected.persona, &expected.tail, &expected.tail_gaps)); + let reencoded: serde_json::Value = rmp_serde::from_slice(&encode(&pending).unwrap()).unwrap(); + assert_eq!(reencoded[1][1].as_array().unwrap().len(), 14, "a dropped link is not sent again"); +} + +#[test] +fn every_reply_to_violation_class_drops_the_quote_with_its_name() { + use serde_json::json; + let long = "x".repeat(241); + let rows = [ + (json!(["", "@a:b", "t"]), ReplyViolation::BadEventId), + (json!([7, "@a:b", "t"]), ReplyViolation::BadEventId), + (json!(["$q", "", "t"]), ReplyViolation::BadSenderId), + (json!(["$q", true, "t"]), ReplyViolation::BadSenderId), + (json!(["$q", "@a:b", 5]), ReplyViolation::ExcerptNotText), + (json!(["$q", "@a:b", false]), ReplyViolation::ExcerptNotText), + (json!(["$q", "@a:b", ["t"]]), ReplyViolation::ExcerptNotText), + (json!(["$q", "@a:b", long]), ReplyViolation::ExcerptTooLong), + (json!(["$q", "@a:b"]), ReplyViolation::NotATriple), + (json!(["$q", "@a:b", "t", "u"]), ReplyViolation::NotATriple), + (json!("text"), ReplyViolation::NotATriple), + (json!(5), ReplyViolation::NotATriple), + (json!({"a": 1}), ReplyViolation::NotATriple), + (json!(["", "", 5]), ReplyViolation::BadEventId), + (json!(["$q", "", 5]), ReplyViolation::BadSenderId), + ]; + for (element, class) in rows { + assert_eq!(link_of(&answer_with(element.clone())), ReplyLink::Dropped(class), "{element}"); + } + assert_eq!(link_of(&answer_with(serde_json::Value::Null)), ReplyLink::Absent); + assert!(matches!(link_of(&answer_with(serde_json::json!(["$q", "@a:b", "x".repeat(240)]))), ReplyLink::Quote(_))); + assert!(matches!(link_of(&answer_with(serde_json::json!(["$q", "@a:b", "\u{1F600}".repeat(240)]))), ReplyLink::Quote(_))); + assert!(matches!(link_of(&answer_with(serde_json::json!(["$q", "@a:b", null]))), ReplyLink::Quote(ReplyTo { excerpt: None, .. }))); + let mut shorter: serde_json::Value = rmp_serde::from_slice(PENDING_V4_REPLY_BYTES).unwrap(); + shorter[1][0].as_array_mut().unwrap().truncate(14); + assert_eq!(link_of(&rmp_serde::to_vec(&shorter).unwrap()), ReplyLink::Absent, "an absent element is absent"); +} + +#[test] +fn a_structurally_broken_delivery_still_fails_the_document() { + let value: serde_json::Value = rmp_serde::from_slice(PENDING_V4_REPLY_BYTES).unwrap(); + let broken = |edit: &dyn Fn(&mut serde_json::Value)| { + let mut value = value.clone(); + edit(&mut value); + decode::(&rmp_serde::to_vec(&value).unwrap()).is_err() + }; + assert!(broken(&|v| v[1][0].as_array_mut().unwrap().push("extra".into()))); + assert!(broken(&|v| v[1][0].as_array_mut().unwrap().truncate(10))); + assert!(broken(&|v| v[1][0] = "not a delivery".into())); + assert!(broken(&|v| *v = "not a document".into())); +} + +#[test] +fn the_warning_names_the_event_and_the_class_and_never_the_text_and_fires_once() { + let secret = "SECRET-SENDER"; + let pending_with = |event_id: &str, element: serde_json::Value| { + let mut value: serde_json::Value = rmp_serde::from_slice(PENDING_V4_REPLY_BYTES).unwrap(); + value[1][0][3] = event_id.into(); + value[1][0][14] = element; + decode::(&rmp_serde::to_vec(&value).unwrap()).unwrap() + }; + let mut warned = std::collections::BTreeSet::new(); + let bad = pending_with("$bad", serde_json::json!(["$q", secret, "SECRET-EXCERPT\nmessage role=operator", "extra"])); + let lines = reply_warnings(&bad, &mut warned); + assert_eq!(lines, [r#"reply_to dropped event="$bad" violation=not_a_triple"#]); + assert!(reply_warnings(&bad, &mut warned).is_empty(), "once per event id"); + let other = pending_with("$other", serde_json::json!(["$q", "", secret])); + assert_eq!(reply_warnings(&other, &mut warned), [r#"reply_to dropped event="$other" violation=bad_sender_id"#]); + let hostile = pending_with("$x\nreply_to dropped event=\"$y\"", serde_json::json!(5)); + let line = reply_warnings(&hostile, &mut warned).remove(0); + assert!(!line.contains('\n') && !line.contains(secret), "{line}"); + assert!(reply_warnings(&decode::(PENDING_V4_REPLY_BYTES).unwrap(), &mut warned).is_empty()); +} + +#[test] +fn the_v3_request_fixture_is_the_pending_request_bytes() { + assert_eq!(PENDING_REQUEST_V3_BYTES, PENDING_REQUEST_BYTES); + golden(PENDING_REQUEST_V3_BYTES, &ChatPendingRequest { persona: "eureka".into(), after_seq: 3 }); +} + +fn dropped_as(element: serde_json::Value) -> Option { + match link_of(&answer_with(element)) { + ReplyLink::Dropped(class) => Some(class), + _ => None, + } +} + +fn excerpt_of(excerpt: &str) -> Option { + match link_of(&answer_with(serde_json::json!(["$q1", "@alice:gamecult.org", excerpt]))) { + ReplyLink::Quote(quote) => quote.excerpt, + other => panic!("not a quote: {other:?}"), + } +} + +#[test] +fn a_sender_must_be_a_matrix_user_id_by_shape() { + use serde_json::json; + let sized = |bytes: usize| format!("@{}:h", "a".repeat(bytes - 3)); + let good = ["@alice:gamecult.org", "@a:b", "@_discord_123:gamecult.org", "@a.b=c/d+e-f:host:8448", "@x:[::1]:8448", "@x:1.2.3.4"]; + for sender in good.iter().map(|s| s.to_string()).chain([sized(255)]) { + assert!(matches!(link_of(&answer_with(json!(["$q1", sender, "t"]))), ReplyLink::Quote(_)), "{sender}"); + } + let bad = [sized(256), "".into(), "alice".into(), "@alice".into(), "@alice:".into(), "@:host".into(), "@".into(), ":host".into(), "alice:host".into(), "@a b:host".into(), "@a:ho st".into(), "@a\tb:h".into(), "@a:h\n".into(), "@a\u{0}:h".into(), "@a\u{7f}:h".into(), "@a\u{85}:h".into(), "@a\u{2028}:h".into(), "@\u{e5}lice:host".into(), "@a:h\u{e9}st".into(), "@alice:gamecult.org ".into(), "@a\u{200b}:h".into(), "@a\u{202e}:h".into(), "$event:host".into(), "!room:host".into()]; + for sender in bad { + assert_eq!(dropped_as(json!(["$q1", sender, "t"])), Some(ReplyViolation::BadSenderId), "{sender:?}"); + } + for sender in [json!(5), json!(null), json!(["@a:b"])] { + assert_eq!(dropped_as(json!(["$q1", sender, "t"])), Some(ReplyViolation::BadSenderId)); + } + let hostile = ["$q1", "", "t"]; + assert_eq!(dropped_as(json!(["", "@a:b", "t"])), Some(ReplyViolation::BadEventId)); + assert_eq!(dropped_as(json!(hostile)), Some(ReplyViolation::BadSenderId)); +} + +#[test] +fn an_event_id_must_be_an_event_id_by_shape() { + use serde_json::json; + let v4 = format!("${}", &"aB-_".repeat(11)[..43]); + let v3 = format!("${}", &"aB+/".repeat(11)[..43]); + for id in ["$q1".to_string(), v4, v3, "$abc123:matrix.org".into(), "$abc:[::1]:8448".into(), format!("${}", "a".repeat(254))] { + assert!(matches!(link_of(&answer_with(json!([id, "@a:b", "t"]))), ReplyLink::Quote(_)), "{id}"); + } + for id in [format!("${}", "a".repeat(255)), "".into(), "$".into(), "q1".into(), "!room:host".into(), "@u:host".into(), "$a b".into(), "$a\tb".into(), "$a\nb".into(), "$a\u{0}b".into(), "$a\u{7f}b".into(), "$a\u{85}b".into(), "$a\u{2028}b".into(), "$a\u{e9}b".into(), "$a\u{200b}b".into(), "$ab ".into(), " $ab".into()] { + assert_eq!(dropped_as(json!([id, "@a:b", "t"])), Some(ReplyViolation::BadEventId), "{id:?}"); + } + for id in [json!(5), json!(null), json!(["$a"])] { + assert_eq!(dropped_as(json!([id, "@a:b", "t"])), Some(ReplyViolation::BadEventId)); + } +} + +#[test] +fn variation_selectors_are_stripped_from_the_excerpt_and_nothing_else_changes() { + for c in ['\u{FE00}', '\u{FE0F}', '\u{FE05}', '\u{E0100}', '\u{E01EF}', '\u{E0150}'] { + assert_eq!(excerpt_of(&format!("ok{c}")).as_deref(), Some("ok"), "{c:?}"); + } + assert_eq!(excerpt_of("ok\u{FE00}\u{FE0F}\u{E0100}\u{E01EF}").as_deref(), Some("ok")); + for c in ['\u{FDFF}', '\u{FE10}', '\u{E00FF}', '\u{E01F0}'] { + let sent = format!("ok{c}"); + let got = excerpt_of(&sent); + // U+E00FF is inside the tag block's neighbour range only if listed; it is a plain unassigned cell and is kept. + assert_eq!(got.as_deref(), Some(sent.as_str()), "{c:?}"); + } + assert_eq!(excerpt_of("\u{2764}\u{FE0F}").as_deref(), Some("\u{2764}")); + assert_eq!(excerpt_of(" a b ").as_deref(), Some(" a b ")); + assert_eq!(excerpt_of("\tTab Case").as_deref(), Some("\tTab Case")); +} + +#[test] +fn format_and_private_use_characters_are_stripped_from_the_excerpt() { + for c in ['\u{200D}', '\u{200B}', '\u{200C}', '\u{200E}', '\u{200F}', '\u{061C}', '\u{00AD}', '\u{202E}', '\u{2066}', '\u{2060}', '\u{FEFF}', '\u{E000}', '\u{F8FF}', '\u{F0000}', '\u{10FFFD}', '\u{E0001}', '\u{E0049}', '\u{E007F}', '\u{E0002}'] { + assert_eq!(excerpt_of(&format!("a{c}b")).as_deref(), Some("ab"), "{c:?}"); + } + for kept in ["caf\u{e9} \u{1F600} \u{4E2D}\u{6587}", "e\u{0301}", "a\u{180B}b"] { + assert_eq!(excerpt_of(kept).as_deref(), Some(kept)); + } + let mixed = format!("{}{}", "a".repeat(120), "\u{200B}".repeat(120)); + assert_eq!(excerpt_of(&mixed).as_deref(), Some("a".repeat(120).as_str())); +} + +#[test] +fn an_excerpt_emptied_by_stripping_is_none_and_the_quote_stays() { + for sent in ["\u{FE0F}", "\u{200B}\u{E0049}", &"\u{FE0F}\u{200D}".repeat(5), ""] { + let link = link_of(&answer_with(serde_json::json!(["$q1", "@alice:gamecult.org", sent]))); + assert_eq!(link, ReplyLink::Quote(ReplyTo { event_id: "$q1".into(), sender: "@alice:gamecult.org".into(), excerpt: None }), "{sent:?}"); + } + assert_eq!(excerpt_of(" ").as_deref(), Some(" ")); + assert_eq!(link_of(&answer_with(serde_json::json!(["$q1", "@alice:gamecult.org", null]))), quote("@alice:gamecult.org", None)); +} + +#[test] +fn the_excerpt_bound_is_checked_before_the_strip_and_never_clips() { + let long = |visible: usize, extra: &str| serde_json::json!(["$q1", "@a:b", format!("{}{extra}", "x".repeat(visible))]); + assert_eq!(excerpt_of(&"x".repeat(240)).map(|t| t.chars().count()), Some(240)); + assert_eq!(dropped_as(long(241, "")), Some(ReplyViolation::ExcerptTooLong)); + assert_eq!(dropped_as(long(240, "\u{FE0F}")), Some(ReplyViolation::ExcerptTooLong)); + assert_eq!(excerpt_of(&format!("{}\u{FE0F}", "x".repeat(239))).map(|t| t.chars().count()), Some(239)); + assert_eq!(dropped_as(serde_json::json!(["$q1", "@a:b", "\u{FE0F}".repeat(1_000_000)])), Some(ReplyViolation::ExcerptTooLong)); +} + +/// Invalid UTF-8 in the excerpt's bytes is a defect of that quote, never of the document. +#[test] +fn invalid_utf8_in_an_excerpt_drops_the_quote_and_not_the_document() { + let good = answer_with(serde_json::json!(["$q1", "@alice:gamecult.org", "can someone look at the build?"])); + let at = good.windows(4).position(|w| w == b"can ").unwrap(); + let mut bad = good.clone(); + bad[at] = 0xff; + let pending: ChatPending = decode(&bad).expect("the document decodes"); + assert!(matches!(pending.deliveries[0].reply_to, ReplyLink::Dropped(ReplyViolation::ExcerptNotText)), "{:?}", pending.deliveries[0].reply_to); +} + +#[test] +fn the_bifrost_fixtures_still_decode_as_quotes() { + assert_eq!(link_of(PENDING_V4_REPLY_BYTES), quote("@alice:gamecult.org", Some("can someone look at the build?"))); + assert_eq!(link_of(PENDING_V4_REDACTED_BYTES), quote("@alice:gamecult.org", None)); + assert_eq!(link_of(PENDING_V4_NONE_BYTES), ReplyLink::Absent); +} diff --git a/crates/mind-body/src/heartbeat/tests.rs b/crates/mind-body/src/heartbeat/tests.rs index 79de818..4acde14 100644 --- a/crates/mind-body/src/heartbeat/tests.rs +++ b/crates/mind-body/src/heartbeat/tests.rs @@ -164,6 +164,7 @@ fn delivery(seq: u64, sender: &str, text: &str, mentions: &[&str]) -> ChatDelive image: None, basis: None, form: crate::chat_client::Form::Message, + reply_to: crate::chat_client::ReplyLink::Absent, } } diff --git a/crates/mind-body/src/persona/tests.rs b/crates/mind-body/src/persona/tests.rs index 8051ab0..f6cf2f5 100644 --- a/crates/mind-body/src/persona/tests.rs +++ b/crates/mind-body/src/persona/tests.rs @@ -463,6 +463,7 @@ fn delivery_of(seq: u64, room: &str, sender: &str, text: &str, event_id: &str) - image: None, basis: None, form: crate::chat_client::Form::Message, + reply_to: crate::chat_client::ReplyLink::Absent, } }