diff --git a/crates/eureka-state/tests/cli.rs b/crates/eureka-state/tests/cli.rs index ba446e5..b0a4614 100644 --- a/crates/eureka-state/tests/cli.rs +++ b/crates/eureka-state/tests/cli.rs @@ -1303,6 +1303,22 @@ impl huginn_daemon::sources::SourceIndex for Desk { anyhow::bail!("a daemon never writes the source index") } + fn chat_tail(&mut self, _room_id: &str) -> anyhow::Result> { + anyhow::bail!("a daemon never reads the ingest job's side of the source index") + } + + fn chat_chunks_holding(&mut self, _room_id: &str, _event_ids: &[String]) -> anyhow::Result> { + anyhow::bail!("a daemon never reads the ingest job's side of the source index") + } + + fn chat_chunks_overlapping(&mut self, _room_id: &str, _from_ms: u64, _to_ms: u64) -> anyhow::Result> { + anyhow::bail!("a daemon never reads the ingest job's side of the source index") + } + + fn delete_chat(&mut self, _room_id: &str, _ids: Option<&[String]>) -> anyhow::Result<()> { + anyhow::bail!("a daemon never writes the source index") + } + fn chat_nearest( &mut self, _vector: &[f32], diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs index 8f4db34..e162bca 100644 --- a/crates/huginn-daemon/src/index/fakes.rs +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -8,7 +8,7 @@ use std::time::Duration; use anyhow::{Result, bail, ensure}; -use crate::sources::{ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta}; +use crate::sources::{ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta, StoredChat}; use super::{CollectionMeta, Described, Embedder, Hit, ModelIdentity, Point, VectorIndex, point_id}; @@ -264,6 +264,16 @@ pub(crate) struct SourcesState { pub(crate) deletes: Vec<(String, Option>)>, /// A delete naming this repo refuses. pub(crate) fail_deletes_of: Option, + /// Every cursor write refuses once this many have been written. + pub(crate) fail_cursor_writes_after: Option, + /// Every chat upsert refuses while this is set. + pub(crate) fail_chat_upserts: bool, + /// Every chat delete refuses while this is set. + pub(crate) fail_chat_deletes: bool, + /// A chat delete naming one of these rooms refuses. + pub(crate) fail_chat_deletes_for: BTreeSet, + /// Every chat delete that went through: the room and the ids (`None` is all of the room). + pub(crate) chat_deletes: Vec<(String, Option>)>, /// The repo filter and limit of every search, in order. pub(crate) searches: Vec<(Option, u32)>, } @@ -326,6 +336,9 @@ impl SourceIndex for FakeSources { fn set_cursors(&mut self, cursors: &BTreeMap) -> Result<()> { let mut state = self.state.lock().unwrap(); + if state.fail_cursor_writes_after.is_some_and(|after| state.cursor_writes.len() >= after) { + bail!("the vector store refused the cursor write"); + } state.cursor_writes.push(cursors.clone()); let Some(DescribedSources::Labelled { cursors: held, .. }) = &mut state.described else { bail!("no source collection") }; *held = cursors.clone(); @@ -375,6 +388,7 @@ impl SourceIndex for FakeSources { fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> { let mut state = self.state.lock().unwrap(); + ensure!(!state.fail_chat_upserts, "the vector store refused the chat upsert"); for point in points { ensure!(!point.payload.messages.is_empty(), "a chat chunk carries no messages"); state.chat.insert(point.id.clone(), point.clone()); @@ -404,6 +418,44 @@ impl SourceIndex for FakeSources { } Ok(kept.into_iter().take(limit as usize).cloned().collect()) } + + fn chat_tail(&mut self, room_id: &str) -> Result> { + let state = self.state.lock().unwrap(); + Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id).max_by_key(|(id, point)| (span(&point.payload).1, (*id).clone())).map(stored)) + } + + fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result> { + let state = self.state.lock().unwrap(); + Ok(state + .chat + .iter() + .filter(|(_, point)| point.payload.room_id == room_id && point.payload.messages.iter().any(|message| event_ids.contains(&message.event_id))) + .map(stored) + .collect()) + } + + fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result> { + let state = self.state.lock().unwrap(); + Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id && span(&point.payload).1 >= from_ms && span(&point.payload).0 < to_ms).map(stored).collect()) + } + + fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> { + let mut state = self.state.lock().unwrap(); + ensure!(!state.fail_chat_deletes && !state.fail_chat_deletes_for.contains(room_id), "the vector store refused the chat delete"); + state.chat_deletes.push((room_id.to_owned(), ids.map(<[String]>::to_vec))); + state.chat.retain(|id, point| !(point.payload.room_id == room_id && ids.is_none_or(|ids| ids.contains(id)))); + Ok(()) + } +} + +fn stored((id, point): (&String, &ChatPoint)) -> StoredChat { + StoredChat { id: id.clone(), payload: point.payload.clone() } +} + +/// (start, end) of the times a chunk's messages make. +fn span(payload: &ChatPayload) -> (u64, u64) { + let times = || payload.messages.iter().map(|message| message.ts_ms); + (times().min().unwrap_or(0), times().max().unwrap_or(0)) } /// A chat chunk under a filter, by the span its messages make. @@ -443,4 +495,16 @@ impl SourceIndex for FakeIndex { fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result> { self.sources.chat_scroll(filter, order, limit, within) } + fn chat_tail(&mut self, room_id: &str) -> Result> { + self.sources.chat_tail(room_id) + } + fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result> { + self.sources.chat_chunks_holding(room_id, event_ids) + } + fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result> { + self.sources.chat_chunks_overlapping(room_id, from_ms, to_ms) + } + fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> { + self.sources.delete_chat(room_id, ids) + } } diff --git a/crates/huginn-daemon/src/index/qdrant.rs b/crates/huginn-daemon/src/index/qdrant.rs index 3f51c0a..3a3f54c 100644 --- a/crates/huginn-daemon/src/index/qdrant.rs +++ b/crates/huginn-daemon/src/index/qdrant.rs @@ -11,7 +11,7 @@ use huginn_mind::ChatMessage; use crate::sources::{ COLLECTION, CORPUS_CHAT, CORPUS_FORGE, ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, FRONT_PAGE_FILES, SourceIndex, SourcePayload, SourcePoint, SourceRow, - SourcesMeta, + SourcesMeta, StoredChat, }; use super::{CollectionMeta, INDEX_CALL_TIMEOUT, Described, Hit, Http, Method, Point, VectorIndex}; @@ -464,6 +464,74 @@ impl SourceIndex for QdrantIndex { let found = answer["result"]["points"].as_array().context("Qdrant's scroll has no points")?; found.iter().map(|point| chat_payload_of(&point["payload"])).collect() } + + fn chat_tail(&mut self, room_id: &str) -> Result> { + let order = json!({ "key": "ts_end_ms", "direction": "desc" }); + let Some(newest) = self.scroll_chat(chat_room_filter(room_id, vec![]), Some((order, 1)))?.into_iter().next() else { return Ok(None) }; + // Chunks tied on the greatest end are all read and the greatest id wins, so the tail is the same chunk on every run. + let end = newest.payload.messages.iter().map(|message| message.ts_ms).max().unwrap_or(0); + let tied = self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "ts_end_ms", "range": { "gte": end } })]), None)?; + Ok(tied.into_iter().max_by(|a, b| a.id.cmp(&b.id))) + } + + fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result> { + if event_ids.is_empty() { + return Ok(Vec::new()); + } + self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "event_ids", "match": { "any": event_ids } })]), None) + } + + fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result> { + let span = vec![json!({ "key": "ts_end_ms", "range": { "gte": from_ms } }), json!({ "key": "ts_start_ms", "range": { "lt": to_ms } })]; + self.scroll_chat(chat_room_filter(room_id, span), None) + } + + fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> { + let collection = self.sources.clone(); + let body = json!({ "filter": chat_delete_filter(room_id, ids) }); + self.ask(Method::Post, &format!("/collections/{collection}/points/delete?wait=true"), Some(&body), &[200])?; + Ok(()) + } +} + +impl QdrantIndex { + /// Every chat chunk `filter` keeps, with its point id, paged; or, given an + /// order and a limit, the first of them in that order. + fn scroll_chat(&mut self, filter: Value, ordered: Option<(Value, u64)>) -> Result> { + let collection = self.sources.clone(); + let mut out = Vec::new(); + let mut offset = Value::Null; + loop { + let mut body = json!({ "filter": filter, "limit": SCROLL_PAGE, "with_payload": true, "with_vector": false }); + if let Some((order, limit)) = &ordered { + body["order_by"] = order.clone(); + body["limit"] = json!(limit); + } else if !offset.is_null() { + body["offset"] = offset; + } + let (_, answer) = self.ask(Method::Post, &format!("/collections/{collection}/points/scroll"), Some(&body), &[200])?; + for point in answer["result"]["points"].as_array().context("Qdrant's scroll has no points")? { + out.push(StoredChat { id: point["id"].as_str().context("a point id is not a string")?.to_owned(), payload: chat_payload_of(&point["payload"])? }); + } + offset = answer["result"]["next_page_offset"].clone(); + if ordered.is_some() || offset.is_null() { + return Ok(out); + } + } + } +} + +/// What a chat delete removes: the room's chat points, all of them or those with `ids`. The corpus +/// and room conditions hold either way, so a forge point or another room's point cannot match. +fn chat_delete_filter(room_id: &str, ids: Option<&[String]>) -> Value { + chat_room_filter(room_id, ids.map(|ids| json!({ "has_id": ids })).into_iter().collect()) +} + +/// The chat points of one room, narrowed by `more` conditions. +fn chat_room_filter(room_id: &str, more: Vec) -> Value { + let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_CHAT } }), json!({ "key": "room_id", "match": { "value": room_id } })]; + must.extend(more); + json!({ "must": must }) } #[cfg(test)] @@ -526,6 +594,29 @@ mod source_filter_tests { assert!(chat_payload_of(&forge).is_err()); } + /// The writer-side filters name the chat corpus and the room, so a forge point never matches. + #[test] + fn the_writer_filters_are_always_chat_corpus_and_the_room() { + let filter = chat_room_filter("!r:x", vec![json!({ "key": "event_ids", "match": { "any": ["$a"] } })]); + assert_eq!( + filter, + json!({ "must": [ + { "key": "corpus", "match": { "value": "chat" } }, + { "key": "room_id", "match": { "value": "!r:x" } }, + { "key": "event_ids", "match": { "any": ["$a"] } }, + ] }) + ); + } + + #[test] + fn a_chat_delete_names_the_corpus_and_room_with_or_without_ids() { + let all = chat_delete_filter("!r:x", None); + assert_eq!(all["must"].as_array().unwrap().len(), 2); + let some = chat_delete_filter("!r:x", Some(&["p1".to_owned()])); + assert_eq!(some["must"][2], json!({ "has_id": ["p1"] })); + assert_eq!(some["must"][0], json!({ "key": "corpus", "match": { "value": "chat" } })); + } + /// The chat filter is the chat corpus, the rooms as an any-of `should` split /// by id and alias, and the two ranges that keep an overlapping span. #[test] diff --git a/crates/huginn-daemon/src/sources/chat.rs b/crates/huginn-daemon/src/sources/chat.rs new file mode 100644 index 0000000..fcee66f --- /dev/null +++ b/crates/huginn-daemon/src/sources/chat.rs @@ -0,0 +1,638 @@ +//! Chat ingest: joined-room history into the shared source collection, beside +//! the forge's repos. `refresh_chat` is the only writer of `corpus=chat` points +//! and of the cursors object's `chat:` keys; `chat_purge::request_purge` is the +//! only writer of `chat-purge:` keys, and `refresh_chat` the only applier. +//! +//! The Matrix client that implements `RoomHistory` is the next cut's and holds no +//! policy. What is in scope is `room_scope`'s; what may be embedded is +//! `huginn_mind::redact::scrub`'s; this file folds events into chunks and writes +//! them, write-then-delete and cursor-last, so any crash replays to the same state. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::time::Instant; + +use anyhow::{Context, Result, bail, ensure}; +use huginn_mind::ChatMessage; +use huginn_mind::redact::{ScrubKind, scrub}; + +use super::chat_chunk::{chat_version, chunk_id, chunk_messages, chunk_payload, cut_body}; +use super::chat_purge::{PurgeWindow, covers, parse_windows, purge_key}; +use super::room_scope::{Estate, Refusal, RoomFacts, admit_room, refuse_event}; +use super::{CORPUS, ChatPoint, DescribedSources, SourceIndex, StoredChat}; +use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Embedder, INDEX_CALL_TIMEOUT}; + +/// Events asked of the port per page. +pub const CHAT_PAGE: u32 = 200; + +/// A chat cursor is the key `chat:`; a forge repo name cannot contain +/// a colon. The only definition of the prefix. +pub const CHAT_CURSOR_PREFIX: &str = "chat:"; + +/// A steward's purge tombstone is the key `chat-purge:`. The only +/// definition of the prefix. +pub const CHAT_PURGE_PREFIX: &str = "chat-purge:"; + +/// Whether a cursors-object key is chat's (a room cursor or a purge tombstone) +/// and not a forge repo name. The only test that tells the two apart. +pub fn is_chat_key(key: &str) -> bool { + key.starts_with(CHAT_CURSOR_PREFIX) || key.starts_with(CHAT_PURGE_PREFIX) +} + +/// What one room event is to the ingest. Only text, notice and emote messages +/// become `Message`; everything else (encrypted, media, state, server-redacted) +/// is `Other` and is never stored. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EventKind { + Message { body: String }, + /// A replacement of `target`'s body. + Edit { target: String, body: String }, + Redaction { target: String }, + Other, +} + +/// One event of a room, as the port hands it over: the Matrix client maps its +/// JSON to this and filters nothing. `content_keys` are the top-level keys of +/// the event's content (the scope reads them). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoomEvent { + pub event_id: String, + pub sender: String, + pub ts_ms: u64, + pub content_keys: Vec, + pub kind: EventKind, +} + +/// One page of a room's events after a cursor event, oldest first. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EventPage { + pub events: Vec, + /// More events follow this page. + pub more: bool, +} + +/// The reader user's view of Synapse as ingest uses it. Holds no policy: it maps +/// the server's answers to these types and filters nothing. +pub trait RoomHistory { + /// The rooms the reader has joined. The only source of rooms; there is no allow list. + fn joined_rooms(&mut self) -> Result>; + fn facts(&mut self, room_id: &str) -> Result; + /// Up to `limit` events after the event `after` (from the room's start when `None`). + /// `Ok(None)` means `after` is unknown to the server. + fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result>; +} + +/// The cursors-object key of a room's chat cursor. +pub fn cursor_key(room_id: &str) -> String { + format!("{CHAT_CURSOR_PREFIX}{room_id}") +} + +/// The room a chat cursor key belongs to; `None` for any other key (a purge tombstone included). +pub fn cursor_room(key: &str) -> Option<&str> { + key.strip_prefix(CHAT_CURSOR_PREFIX) +} + +/// A cursor value, `:`; the event is empty before a room's first upsert. +pub fn parse_cursor(value: &str) -> Option<(&str, &str)> { + value.split_once(':') +} + +/// What one admitted room's run did, for the journal. Never carries a body or a sender. +#[derive(Clone, Debug, PartialEq)] +pub struct ChatReport { + pub room_id: String, + pub alias: Option, + pub first: Option, + pub last: Option, + pub events: u32, + pub chunks: u32, + pub scrubbed: u32, + pub purged: u32, + pub seconds: f64, + /// Why the room could not be read or written this run; what the index held for it stays as it was. + pub failure: Option, +} + +impl std::fmt::Display for ChatReport { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "{} ({}) {}..{} events={} chunks={} scrubbed={} purged={} seconds={:.1}", + self.alias.as_deref().unwrap_or("-"), + self.room_id, + self.first.as_deref().unwrap_or("-"), + self.last.as_deref().unwrap_or("-"), + self.events, + self.chunks, + self.scrubbed, + self.purged, + self.seconds + )?; + match &self.failure { + Some(why) => write!(formatter, " failed: {why}"), + None => Ok(()), + } + } +} + +/// A joined room the scope refused, whole, and why; `event_id` when an event did it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RefusedRoom { + pub room_id: String, + pub reason: Refusal, + pub event_id: Option, +} + +/// A credential the scrub replaced: where, never what. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScrubHit { + pub room_id: String, + pub event_id: String, + pub kind: ScrubKind, +} + +/// What a chat refresh did. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ChatRefreshed { + pub rooms: Vec, + /// Rooms (or steps) that failed; the others went on. + pub failed: Vec, + pub refused: Vec, + pub scrubbed: Vec, +} + +impl ChatReport { + fn empty(room_id: &str, alias: Option) -> Self { + Self { room_id: room_id.to_owned(), alias, first: None, last: None, events: 0, chunks: 0, scrubbed: 0, purged: 0, seconds: 0.0, failure: None } + } + + /// A room that could not be handled this run. Its points and cursor are untouched: an error is not a reclassification. + fn failed(room_id: &str, alias: Option, error: &anyhow::Error) -> Self { + Self { failure: Some(format!("{error:#}")), ..Self::empty(room_id, alias) } + } +} + +impl ChatRefreshed { + /// Nothing failed and no joined room had to be refused: a joined estate, direct, + /// encrypted or bridged room means the reader's roster is wrong. A scrub hit does not make a run unclean. + pub fn is_clean(&self) -> bool { + self.failed.is_empty() && self.refused.is_empty() + } +} + +fn read_cursors(index: &mut S) -> Result> { + match index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? { + DescribedSources::Labelled { cursors, .. } => Ok(cursors), + _ => bail!("the source collection is gone; the forge refresh creates it"), + } +} + +fn put_cursor(index: &mut S, key: &str, value: &str) -> Result<()> { + let mut cursors = read_cursors(index)?; + if cursors.insert(key.to_owned(), value.to_owned()).as_deref() != Some(value) { + index.set_cursors(&cursors).context("writing a chat cursor")?; + } + Ok(()) +} + +fn drop_cursor(index: &mut S, key: &str) -> Result<()> { + let mut cursors = read_cursors(index)?; + if cursors.remove(key).is_some() { + index.set_cursors(&cursors).context("dropping a chat cursor")?; + } + Ok(()) +} + +/// A room's points, then its `chat:` cursor. Its purge tombstone stays. +fn purge_room(index: &mut S, room_id: &str) -> Result<()> { + index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id}"))?; + drop_cursor(index, &cursor_key(room_id)) +} + +/// What `retire_chat` did: the rooms it removed, and those it could not (each with why), which it went on past. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Retired { + pub retired: Vec, + pub failed: Vec<(String, String)>, +} + +/// Deletes the points and `chat:` cursor of every room not in `keep` (the reader left it, +/// or it is refused or excluded). A room whose delete fails is reported and the others go on. Purge +/// tombstones and forge repos are never touched. With the empty set it removes the whole chat corpus. +pub fn retire_chat(index: &mut S, keep: &BTreeSet) -> Result { + let rooms: Vec = read_cursors(index)?.keys().filter_map(|key| cursor_room(key)).filter(|room| !keep.contains(*room)).map(str::to_owned).collect(); + let mut done = Retired::default(); + for room in rooms { + match purge_room(index, &room) { + Ok(()) => done.retired.push(room), + Err(error) => done.failed.push((room, format!("{error:#}"))), + } + } + Ok(done) +} + +/// A change to chunks already stored, in event order. +enum Op { + /// An edit by `sender` of `target`'s body; applies only to the original's sender. + Edit { id: String, target: String, sender: String, body: String }, + /// A held message's body is what the page says it is (a replay). + Set { target: String, body: String }, + Redact { target: String }, +} + +impl Op { + fn target(&self) -> &str { + match self { + Self::Edit { target, .. } | Self::Set { target, .. } | Self::Redact { target } => target, + } + } + + /// Applies to `messages`; whether they changed. + fn apply(&self, messages: &mut Vec) -> bool { + let Some(at) = messages.iter().position(|message| message.event_id == self.target()) else { return false }; + match self { + Self::Edit { sender, body, .. } if messages[at].sender == *sender && messages[at].body != *body => messages[at].body.clone_from(body), + Self::Set { body, .. } if messages[at].body != *body => messages[at].body.clone_from(body), + Self::Redact { .. } => { + messages.remove(at); + } + _ => return false, + } + true + } +} + +/// A message of the page with the edits of the page that apply to it: the body is the last edit's, and a +/// redacted edit leaves the list, so the body falls back to the one before it. +struct Pending { + message: ChatMessage, + original: String, + edits: Vec<(String, String)>, +} + +struct Folded { + pending: Vec, + ops: Vec, + /// Every event id of the page: a redaction of one of them is known. + seen: HashSet, +} + +struct Room<'a> { + facts: &'a RoomFacts, + label: String, + windows: Vec, +} + +/// The page folded in order: the purge filter, then the credential scrub, then the +/// event's effect. After this no raw body exists. +fn fold(room: &Room, events: &[RoomEvent], report: &mut ChatReport, out: &mut ChatRefreshed) -> Folded { + let mut pending: Vec = Vec::new(); + let mut ops: Vec = Vec::new(); + let mut scrubbed = |event: &RoomEvent, body: &str| { + let clean = scrub(body); + for kind in clean.hits { + report.scrubbed += 1; + out.scrubbed.push(ScrubHit { room_id: room.facts.room_id.clone(), event_id: event.event_id.clone(), kind }); + } + cut_body(&clean.text) + }; + for event in events { + match &event.kind { + EventKind::Message { body } => { + if covers(&room.windows, event.ts_ms) { + report.purged += 1; + } else { + let body = scrubbed(event, body); + if !body.trim().is_empty() { + let message = ChatMessage { event_id: event.event_id.clone(), sender: event.sender.clone(), ts_ms: event.ts_ms, body: body.clone() }; + pending.push(Pending { message, original: body, edits: Vec::new() }); + } + } + } + EventKind::Edit { target, body } => { + if covers(&room.windows, event.ts_ms) { + report.purged += 1; + continue; + } + let body = scrubbed(event, body); + match pending.iter_mut().find(|held| held.message.event_id == *target) { + Some(held) if held.message.sender == event.sender && !body.trim().is_empty() => { + held.edits.push((event.event_id.clone(), body.clone())); + held.message.body = body; + } + Some(_) => {} + None => ops.push(Op::Edit { id: event.event_id.clone(), target: target.clone(), sender: event.sender.clone(), body }), + } + } + EventKind::Redaction { target } => { + // The target may be a message, or an edit whose own text must stop applying. + pending.retain(|held| held.message.event_id != *target); + for held in &mut pending { + held.edits.retain(|(id, _)| id != target); + held.message.body = held.edits.last().map_or_else(|| held.original.clone(), |(_, body)| body.clone()); + } + ops.retain(|op| !matches!(op, Op::Edit { id, .. } if id == target)); + ops.push(Op::Redact { target: target.clone() }); + } + EventKind::Other => {} + } + } + Folded { pending: pending.into_iter().map(|held| held.message).collect(), ops, seen: events.iter().map(|event| event.event_id.clone()).collect() } +} + +/// Chunks to rewrite: the stored chunk ids they replace and the messages that now make them. +type Rewrite = (Vec, Vec); + +/// Chunks `rewrites`, embeds and writes the new chunks (acknowledged), and only then deletes the old +/// ids that are not among the new. Returns how many chunks were written. +/// +/// A chunk written by an earlier run whose delete failed can still sit beside the chunk that replaced +/// it, and a rewrite of the stale one then names the same first event, hence the same id, as the +/// newer chunk. The newer chunk (`standing`, the other chunks the page found) stays; the stale one is deleted. +fn write_rewrites( + embedder: &mut E, + index: &mut S, + dimensions: u32, + room: &Room, + rewrites: Vec, + standing: &BTreeSet, +) -> Result { + let mut old: BTreeSet = BTreeSet::new(); + let mut points: Vec<(String, super::ChatPayload)> = Vec::new(); + for (ids, messages) in rewrites { + for group in chunk_messages(&room.label, &messages) { + let id = chunk_id(&room.facts.room_id, &group[0].event_id); + if standing.contains(&id) && !ids.contains(&id) { + continue; + } + points.push((id, chunk_payload(&room.facts.room_id, room.facts.alias.as_deref(), &room.label, group))); + } + old.extend(ids); + } + let new: BTreeSet<&String> = points.iter().map(|(id, _)| id).collect(); + let stale: Vec = old.iter().filter(|id| !new.contains(id)).cloned().collect(); + let written = u32::try_from(points.len()).unwrap_or(u32::MAX); + for slice in points.chunks(BATCH) { + let texts: Vec = slice.iter().map(|(_, payload)| payload.text.clone()).collect(); + let vectors = embedder.embed(&texts, BULK_EMBED_TIMEOUT).context("embedding chat chunks")?; + ensure!(vectors.len() == texts.len(), "the embedder returned {} vectors for {} chunks", vectors.len(), texts.len()); + let mut batch = Vec::with_capacity(slice.len()); + for ((id, payload), vector) in slice.iter().zip(vectors) { + ensure!(vector.len() == dimensions as usize, "the embedder returned a vector of {} where the collection holds {dimensions}", vector.len()); + batch.push(ChatPoint { id: id.clone(), vector, payload: payload.clone() }); + } + index.upsert_chat(&batch).context("writing chat chunks")?; + } + if !stale.is_empty() { + index.delete_chat(&room.facts.room_id, Some(&stale)).context("deleting superseded chat chunks")?; + } + Ok(written) +} + +/// Applies the room's purge windows to the chunks it already holds. +fn apply_purge(embedder: &mut E, index: &mut S, dimensions: u32, room: &Room, report: &mut ChatReport) -> Result<()> { + let mut hit: BTreeMap = BTreeMap::new(); + for window in &room.windows { + for chunk in index.chat_chunks_overlapping(&room.facts.room_id, window.from_ms, window.to_ms)? { + hit.insert(chunk.id.clone(), chunk); + } + } + let mut rewrites = Vec::new(); + for chunk in hit.into_values() { + let kept: Vec = chunk.payload.messages.iter().filter(|message| !covers(&room.windows, message.ts_ms)).cloned().collect(); + let removed = chunk.payload.messages.len() - kept.len(); + if removed > 0 { + report.purged += u32::try_from(removed).unwrap_or(u32::MAX); + rewrites.push((vec![chunk.id], kept)); + } + } + report.chunks += write_rewrites(embedder, index, dimensions, room, rewrites, &BTreeSet::new())?; + Ok(()) +} + +/// Folds one page into the stored chunks: edits and redactions rewrite the chunk that held +/// the target alone, new messages extend the open tail. +fn write_page( + embedder: &mut E, + index: &mut S, + dimensions: u32, + room: &Room, + mut folded: Folded, + may_rebuild: bool, +) -> Result> { + let room_id = &room.facts.room_id; + let mut asked: Vec = folded.pending.iter().map(|message| message.event_id.clone()).collect(); + asked.extend(folded.ops.iter().map(|op| op.target().to_owned())); + if asked.is_empty() { + return Ok(Some(0)); + } + let holding = index.chat_chunks_holding(room_id, &asked)?; + // A message the index already holds is a replay: its page body is asserted, never appended again. + let held: HashSet<&str> = holding.iter().flat_map(|chunk| chunk.payload.messages.iter().map(|message| message.event_id.as_str())).collect(); + // A redaction of an event this room neither holds nor shows on the page may be of an edit whose own id is + // never stored, so a body it changed cannot be taken back here: the room is rebuilt from the history, which + // serves a redacted edit without its text. + if may_rebuild && folded.ops.iter().any(|op| matches!(op, Op::Redact { target } if !folded.seen.contains(target) && !held.contains(target.as_str()))) { + return Ok(None); + } + let (replayed, fresh): (Vec, Vec) = folded.pending.drain(..).partition(|message| held.contains(message.event_id.as_str())); + let mut ops: Vec = replayed.into_iter().map(|message| Op::Set { target: message.event_id, body: message.body }).collect(); + ops.append(&mut folded.ops); + + let tail = index.chat_tail(room_id)?; + let tail_id = tail.as_ref().map(|tail| tail.id.clone()); + let standing: BTreeSet = holding.iter().map(|chunk| chunk.id.clone()).chain(tail_id.clone()).collect(); + let mut tail_messages = tail.map(|tail| tail.payload.messages).unwrap_or_default(); + let mut tail_changed = false; + let mut rewrites: Vec = Vec::new(); + for chunk in holding { + let mut messages = chunk.payload.messages.clone(); + let mut changed = false; + for op in &ops { + changed |= op.apply(&mut messages); + } + if Some(&chunk.id) == tail_id.as_ref() { + (tail_messages, tail_changed) = (messages, changed); + } else if changed { + rewrites.push((vec![chunk.id], messages)); + } + } + if !fresh.is_empty() || tail_changed { + tail_messages.extend(fresh); + rewrites.push((tail_id.into_iter().collect(), tail_messages)); + } + write_rewrites(embedder, index, dimensions, room, rewrites, &standing).map(Some) +} + +enum Outcome { + Done(ChatReport), + Refused(RefusedRoom), +} + +#[allow(clippy::too_many_arguments)] +fn ingest_room( + embedder: &mut E, + index: &mut S, + history: &mut H, + dimensions: u32, + facts: &RoomFacts, + out: &mut ChatRefreshed, +) -> Result { + let started = Instant::now(); + let room_id = facts.room_id.as_str(); + let (key, version) = (cursor_key(room_id), chat_version()); + let cursors = read_cursors(index)?; + let mut may_rebuild = false; + let mut after: Option = match cursors.get(&key).map(|value| parse_cursor(value)) { + None => None, + Some(Some((held, event))) if held == version => { + may_rebuild = true; + Some(event.to_owned()).filter(|event| !event.is_empty()) + } + // Another chunker or scrub version, or a value nobody here wrote: this room alone starts over. + Some(_) => { + index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id} for a re-ingest"))?; + None + } + }; + if after.is_none() { + put_cursor(index, &key, &format!("{version}:"))?; + } + let windows = cursors.get(&purge_key(room_id)).map(|value| parse_windows(value)).transpose()?.unwrap_or_default(); + let mut report = ChatReport::empty(room_id, facts.alias.clone()); + // The room's name is embedded and stored in every chunk, so it passes the scrub like a body. + let label = match &facts.name { + Some(name) => { + let clean = scrub(name); + for kind in clean.hits { + report.scrubbed += 1; + out.scrubbed.push(ScrubHit { room_id: room_id.to_owned(), event_id: "(name)".to_owned(), kind }); + } + clean.text + } + None => facts.alias.clone().unwrap_or_else(|| room_id.to_owned()), + }; + let room = Room { facts, label, windows }; + apply_purge(embedder, index, dimensions, &room, &mut report)?; + let mut restarted = false; + loop { + let Some(page) = history.events_after(room_id, after.as_deref(), CHAT_PAGE)? else { + ensure!(after.is_some() && !restarted, "the server does not know the start of {room_id}'s history"); + restarted = true; + purge_room(index, room_id)?; + put_cursor(index, &key, &format!("{version}:"))?; + after = None; + continue; + }; + let Some(last) = page.events.last().map(|event| event.event_id.clone()) else { break }; + // A refusing event refuses the room before anything of the page is chunked or embedded. + if let Some((event, reason)) = page.events.iter().find_map(|event| refuse_event(event).map(|reason| (event, reason))) { + purge_room(index, room_id)?; + return Ok(Outcome::Refused(RefusedRoom { room_id: room_id.to_owned(), reason, event_id: Some(event.event_id.clone()) })); + } + report.first.get_or_insert_with(|| page.events[0].event_id.clone()); + report.events += u32::try_from(page.events.len()).unwrap_or(u32::MAX); + let folded = fold(&room, &page.events, &mut report, out); + let Some(written) = write_page(embedder, index, dimensions, &room, folded, may_rebuild)? else { + // Start this room over from the history; nothing it read is trusted to be complete. + purge_room(index, room_id)?; + put_cursor(index, &key, &format!("{version}:"))?; + (after, may_rebuild) = (None, false); + report = ChatReport { scrubbed: report.scrubbed, ..ChatReport::empty(room_id, facts.alias.clone()) }; + continue; + }; + report.chunks += written; + put_cursor(index, &key, &format!("{version}:{last}"))?; + report.last = Some(last.clone()); + after = Some(last); + if !page.more { + break; + } + } + report.seconds = started.elapsed().as_secs_f64(); + Ok(Outcome::Done(report)) +} + +/// Brings the chat corpus to the reader's joined rooms. The collection must already exist and be +/// Huginn's, built under this embedder's model: the forge refresh creates and rebuilds it, and this never does. +/// A joined room the scope refuses is purged and named; a room the reader left is retired; the rest +/// are ingested from their cursors. One room's failure is journaled and the others go on. +/// `journal` receives every line the run prints: reports, scrub hits, refusals and failures. +pub fn refresh_chat( + embedder: &mut E, + index: &mut S, + history: &mut H, + estate: &Estate, + journal: &mut dyn FnMut(String), +) -> Result { + let DescribedSources::Labelled { meta, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else { + bail!("the source collection is absent or not Huginn's; the forge refresh creates it and chat never does"); + }; + ensure!(meta.corpus == CORPUS, "the source collection was built for corpus {} and chat ingest writes {CORPUS}; the forge refresh rebuilds it", meta.corpus); + embedder.built_under(Some(&meta.model_identity())); + let identity = embedder.model_identity(BULK_EMBED_TIMEOUT).context("reading the embedding model's identity")?; + ensure!(meta.model_identity() == identity, "the source collection was built under {} {} and chat embeds under {} {}", meta.model, meta.model_digest, identity.name, identity.digest); + + let mut out = ChatRefreshed::default(); + let mut joined = history.joined_rooms().context("listing the reader's joined rooms")?; + let mut seen = HashSet::new(); + joined.retain(|room| seen.insert(room.clone())); + let mut keep: BTreeSet = joined.iter().cloned().collect(); + let mut admitted: Vec = Vec::new(); + for room_id in &joined { + let verdict = if estate.holds_id(room_id) { + Ok(Err(Refusal::Estate)) + } else { + history.facts(room_id).map(|facts| admit_room(&facts, estate).map(|()| facts)) + }; + match verdict { + Ok(Ok(facts)) => admitted.push(facts), + Ok(Err(reason)) => { + // Not kept, so the retirement below purges whatever it held. + keep.remove(room_id); + journal(format!("refused: {room_id} {reason}")); + out.refused.push(RefusedRoom { room_id: room_id.clone(), reason, event_id: None }); + } + // An error is not a reclassification: the room keeps its points and cursor, and the report says why. + Err(error) => { + let report = ChatReport::failed(room_id, None, &error.context("reading the room's facts")); + journal(report.to_string()); + out.failed.push(room_id.clone()); + out.rooms.push(report); + } + } + } + match retire_chat(index, &keep) { + Ok(done) => { + for (room, why) in done.failed { + journal(format!("huginn-sources: retiring {room} failed, the others go on: {why}")); + out.failed.push(room); + } + } + Err(error) => { + journal(format!("huginn-sources: retiring rooms failed: {error:#}")); + out.failed.push("retire".to_owned()); + } + } + for facts in admitted { + match ingest_room(embedder, index, history, identity.dimensions, &facts, &mut out) { + Ok(Outcome::Done(report)) => { + journal(report.to_string()); + out.rooms.push(report); + } + Ok(Outcome::Refused(refused)) => { + journal(format!("refused: {} {} [{}]", refused.room_id, refused.reason, refused.event_id.as_deref().unwrap_or(""))); + out.refused.push(refused); + } + Err(error) => { + let report = ChatReport::failed(&facts.room_id, facts.alias.clone(), &error); + journal(report.to_string()); + out.failed.push(facts.room_id.clone()); + out.rooms.push(report); + } + } + } + for hit in &out.scrubbed { + journal(format!("scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind)); + } + Ok(out) +} diff --git a/crates/huginn-daemon/src/sources/chat_chunk.rs b/crates/huginn-daemon/src/sources/chat_chunk.rs new file mode 100644 index 0000000..3ccee13 --- /dev/null +++ b/crates/huginn-daemon/src/sources/chat_chunk.rs @@ -0,0 +1,166 @@ +//! The chat chunker: greedy, deterministic and local. A chunk is consecutive +//! messages of one room; it closes at a time gap, at a message count, or when +//! its embedded text would pass a character bound. A message is never split. + +use chrono::DateTime; +use huginn_mind::redact::SCRUB_VERSION; +use huginn_mind::ChatMessage; + +use super::ChatPayload; +use crate::index::point_id; + +/// Bumped whenever chunk boundaries or the embedded text would change for the +/// same messages. Part of `chat_version`. +pub const CHAT_CHUNKER_VERSION: u32 = 1; + +/// A chunk closes when the next message is more than this much later. +pub const CHAT_GAP_MS: u64 = 30 * 60 * 1000; +/// The most messages a chunk holds. +pub const CHAT_MESSAGES_MAX: usize = 24; +/// The most characters a chunk's embedded text holds. +pub const CHAT_CHARS_MAX: usize = 1_600; +/// The most characters a stored message body holds, marker included. +pub const CHAT_BODY_CHARS_MAX: usize = 1_200; + +/// Ends a body that was cut. +const CUT_MARKER: &str = "…[cut]"; + +/// What the room's cursor and chunk ids are stamped with: the chunker and the +/// scrub. Either changing re-ingests the room. +pub fn chat_version() -> String { + format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}") +} + +/// A chunk's id: where it starts and under which version it was cut. +pub fn chunk_id(room_id: &str, first_event_id: &str) -> String { + point_id(&format!("{room_id}\0{first_event_id}\0{}", chat_version())) +} + +/// A scrubbed body cut to `CHAT_BODY_CHARS_MAX` characters on a character boundary, with a marker. +pub fn cut_body(body: &str) -> String { + if body.chars().count() <= CHAT_BODY_CHARS_MAX { + return body.to_owned(); + } + let keep = CHAT_BODY_CHARS_MAX - CUT_MARKER.chars().count(); + body.chars().take(keep).chain(CUT_MARKER.chars()).collect() +} + +fn line_of(message: &ChatMessage) -> String { + let when = DateTime::from_timestamp_millis(i64::try_from(message.ts_ms).unwrap_or(i64::MAX)).map_or_else(String::new, |at| at.format("%Y-%m-%d %H:%M").to_string()); + format!("{when} {}: {}", message.sender, message.body) +} + +/// What is embedded for a chunk: the room's label (its name, else its alias, +/// else its id), then one `YYYY-MM-DD HH:MM sender: body` line per message, UTC. +pub fn embedded_text(label: &str, messages: &[ChatMessage]) -> String { + let mut text = label.to_owned(); + for message in messages { + text.push('\n'); + text.push_str(&line_of(message)); + } + text +} + +/// `messages` (one room, in order) cut into chunks, each a run of consecutive messages. +pub fn chunk_messages(label: &str, messages: &[ChatMessage]) -> Vec> { + let mut chunks: Vec> = Vec::new(); + let mut chars = 0_usize; + for message in messages { + let line = line_of(message).chars().count() + 1; + let closes = chunks.last().is_some_and(|open| { + let last = open.last().map_or(0, |last| last.ts_ms); + message.ts_ms.saturating_sub(last) > CHAT_GAP_MS || open.len() >= CHAT_MESSAGES_MAX || chars + line > CHAT_CHARS_MAX + }); + if closes || chunks.is_empty() { + chunks.push(Vec::new()); + chars = label.chars().count(); + } + chunks.last_mut().expect("a chunk is open").push(message.clone()); + chars += line; + } + chunks +} + +/// The point payload of one chunk. +pub fn chunk_payload(room_id: &str, alias: Option<&str>, label: &str, messages: Vec) -> ChatPayload { + ChatPayload { room_id: room_id.to_owned(), room_alias: alias.map(str::to_owned), text: embedded_text(label, &messages), messages } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn message(n: u64, ts_ms: u64, body: &str) -> ChatMessage { + ChatMessage { event_id: format!("$e{n}"), sender: "@a:s".into(), ts_ms, body: body.into() } + } + + fn ids(chunks: &[Vec]) -> Vec> { + chunks.iter().map(|chunk| chunk.iter().map(|message| message.event_id.as_str()).collect()).collect() + } + + #[test] + fn a_gap_a_count_and_a_character_bound_each_close_a_chunk() { + let gap = [message(1, 0, "a"), message(2, CHAT_GAP_MS, "b"), message(3, 2 * CHAT_GAP_MS + 1, "c")]; + assert_eq!(ids(&chunk_messages("r", &gap)), [["$e1", "$e2"].as_slice(), ["$e3"].as_slice()], "a gap of exactly the bound does not close"); + let many: Vec = (0..25).map(|n| message(n, n, "x")).collect(); + let chunks = chunk_messages("r", &many); + assert_eq!((chunks[0].len(), chunks[1].len()), (CHAT_MESSAGES_MAX, 1)); + let long: Vec = (0..4).map(|n| message(n, n, &"w".repeat(500))).collect(); + let chunks = chunk_messages("r", &long); + assert!(chunks.iter().all(|chunk| embedded_text("r", chunk).chars().count() <= CHAT_CHARS_MAX), "{:?}", ids(&chunks)); + assert!(chunks.len() > 1); + } + + #[test] + fn the_embedded_text_is_the_label_then_utc_lines() { + let text = embedded_text("Swarm", &[message(1, 1_700_000_000_000, "hello")]); + assert_eq!(text, "Swarm\n2023-11-14 22:13 @a:s: hello"); + } + + #[test] + fn a_long_body_is_cut_at_the_bound_with_a_marker_on_a_char_boundary_and_a_short_one_is_whole() { + let cut = cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX + 50)); + assert_eq!(cut.chars().count(), CHAT_BODY_CHARS_MAX); + assert!(cut.ends_with(CUT_MARKER)); + assert_eq!(cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX)), "é".repeat(CHAT_BODY_CHARS_MAX)); + let one = chunk_messages("r", &[message(1, 0, &cut)]); + assert_eq!(one.len(), 1); + } + + #[test] + fn a_chunk_id_is_its_room_first_event_and_version() { + assert_eq!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$a")); + assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b")); + assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!q:s", "$a")); + assert_eq!(chat_version(), format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}")); + } + + /// Chunking the whole sequence equals re-chunking the open tail with each new batch appended. + #[test] + fn appending_to_the_open_tail_gives_the_boundaries_of_chunking_all_at_once() { + let mut seed = 0x2545_f491_4f6c_dd1d_u64; + let mut next = move |bound: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % bound + }; + let mut ts = 0; + let all: Vec = (0..300) + .map(|n| { + ts += if next(7) == 0 { CHAT_GAP_MS + 1 } else { next(60_000) }; + message(n, ts, &"z".repeat(next(700) as usize + 1)) + }) + .collect(); + let whole = chunk_messages("r", &all); + for step in [1_usize, 5, 17, 100] { + let mut held: Vec> = Vec::new(); + for batch in all.chunks(step) { + let mut tail: Vec = held.pop().unwrap_or_default(); + tail.extend_from_slice(batch); + held.extend(chunk_messages("r", &tail)); + } + assert_eq!(held, whole, "step {step}"); + } + } +} diff --git a/crates/huginn-daemon/src/sources/chat_purge.rs b/crates/huginn-daemon/src/sources/chat_purge.rs new file mode 100644 index 0000000..fbc1778 --- /dev/null +++ b/crates/huginn-daemon/src/sources/chat_purge.rs @@ -0,0 +1,103 @@ +//! A steward's purge of a room's time window. The tombstone is the one chat +//! state that is not derived from Synapse: a durable request, kept in the +//! collection's cursors object under `chat-purge:`, that must outlive +//! the points it removes, so a re-ingest never brings the window back. +//! +//! `request_purge` is the only writer; `refresh_chat` reads the tombstone and +//! applies it at its next run. The code does not authenticate the steward: whoever +//! can run the job as the unit's user (the host admin) may ask, and the request +//! carries a ref in the job's journal for a trace. Tombstones live in the +//! collection, so a full collection rebuild forgets them; re-issue them from +//! their refs. + +use anyhow::{Context, Result, bail}; + +use super::chat::CHAT_PURGE_PREFIX; +use super::{DescribedSources, SourceIndex}; +use crate::index::INDEX_CALL_TIMEOUT; + +/// The most windows a room's tombstone holds. +pub const PURGE_WINDOWS_MAX: usize = 32; + +/// Messages with `from_ms <= ts_ms < to_ms` are purged. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct PurgeWindow { + pub from_ms: u64, + pub to_ms: u64, +} + +impl PurgeWindow { + pub fn covers(&self, ts_ms: u64) -> bool { + self.from_ms <= ts_ms && ts_ms < self.to_ms + } +} + +/// The tombstone key of a room. +pub fn purge_key(room_id: &str) -> String { + format!("{CHAT_PURGE_PREFIX}{room_id}") +} + +/// A tombstone value, `..[,.....]`. +pub fn parse_windows(value: &str) -> Result> { + value + .split(',') + .map(|part| { + let (from, to) = part.split_once("..").with_context(|| format!("purge window {part:?} is not .."))?; + let window = PurgeWindow { from_ms: from.parse().context("a purge window's start is not milliseconds")?, to_ms: to.parse().context("a purge window's end is not milliseconds")? }; + if window.from_ms >= window.to_ms { + bail!("purge window {part:?} is empty"); + } + Ok(window) + }) + .collect() +} + +/// Windows sorted, with overlapping and touching ones joined. +pub fn merge(mut windows: Vec) -> Vec { + windows.sort(); + let mut merged: Vec = Vec::new(); + for window in windows { + match merged.last_mut() { + Some(last) if window.from_ms <= last.to_ms => last.to_ms = last.to_ms.max(window.to_ms), + _ => merged.push(window), + } + } + merged +} + +pub fn encode(windows: &[PurgeWindow]) -> String { + windows.iter().map(|window| format!("{}..{}", window.from_ms, window.to_ms)).collect::>().join(",") +} + +/// Whether any window covers `ts_ms`. +pub fn covers(windows: &[PurgeWindow], ts_ms: u64) -> bool { + windows.iter().any(|window| window.covers(ts_ms)) +} + +/// Records that `window` of `room_id` must leave the index and never return. Writes the cursors +/// object only if the tombstone changed, and refuses a room's 33rd window. Writes no point and +/// embeds nothing; the next `refresh_chat` applies it. The job's `chat-purge` subcommand calls +/// it under the job's lock, because `set_cursors` replaces the cursors object whole. +/// Returns whether the tombstone changed. +pub fn request_purge(index: &mut S, room_id: &str, window: PurgeWindow) -> Result { + if window.from_ms >= window.to_ms { + bail!("a purge window must start before it ends"); + } + let DescribedSources::Labelled { mut cursors, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else { + bail!("the source collection is absent or not Huginn's; the forge refresh creates it"); + }; + let key = purge_key(room_id); + let mut windows = cursors.get(&key).map(|held| parse_windows(held)).transpose()?.unwrap_or_default(); + windows.push(window); + let windows = merge(windows); + if windows.len() > PURGE_WINDOWS_MAX { + bail!("room {room_id} already holds {PURGE_WINDOWS_MAX} purge windows; widen one instead"); + } + let encoded = encode(&windows); + if cursors.get(&key) == Some(&encoded) { + return Ok(false); + } + cursors.insert(key, encoded); + index.set_cursors(&cursors).context("writing the purge tombstone")?; + Ok(true) +} diff --git a/crates/huginn-daemon/src/sources/chat_tests.rs b/crates/huginn-daemon/src/sources/chat_tests.rs new file mode 100644 index 0000000..838d3fe --- /dev/null +++ b/crates/huginn-daemon/src/sources/chat_tests.rs @@ -0,0 +1,964 @@ +//! Chat ingest against a scripted room history, a fake vector store and the fake embedder. +//! The refusals come first: what must never be read, embedded or stored. + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::Duration; + +use anyhow::{Result, bail}; +use huginn_mind::ChatMessage; +use huginn_mind::redact::ScrubKind; + +use super::chat::*; +use super::chat_chunk::{CHAT_BODY_CHARS_MAX, chat_version, chunk_id}; +use super::chat_purge::{PURGE_WINDOWS_MAX, PurgeWindow, request_purge}; +use super::room_scope::{ESTATE_ALIASES, Estate, Refusal, RoomFacts}; +use super::*; +use crate::index::MANAGED_BY; +use crate::index::fakes::{FakeEmbedder, FakeSources}; + +// ---- the rig ---- + +fn plain(id: &str) -> RoomFacts { + RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None } +} + +fn estate(ids: &[&str]) -> Estate { + Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap() +} + +struct Scripted { + facts: RoomFacts, + events: Vec, +} + +#[derive(Default)] +struct FakeHistory { + joined: Vec, + rooms: BTreeMap, + calls: Vec, + /// Rooms whose state cannot be read. + facts_fail: BTreeSet, + /// Serve an event a redaction targets as the server does, without its content. + prune: bool, +} + +impl FakeHistory { + fn room(&mut self, id: &str, events: Vec) -> &mut Self { + self.rooms.insert(id.into(), Scripted { facts: plain(id), events }); + self.joined.push(id.into()); + self + } + + fn facts(&mut self, id: &str, edit: impl FnOnce(&mut RoomFacts)) -> &mut Self { + edit(&mut self.rooms.get_mut(id).unwrap().facts); + self + } + + fn push(&mut self, id: &str, events: Vec) { + self.rooms.get_mut(id).unwrap().events.extend(events); + } + + fn called(&self, prefix: &str, room: &str) -> bool { + self.calls.iter().any(|call| call == &format!("{prefix}:{room}")) + } +} + +impl RoomHistory for FakeHistory { + fn joined_rooms(&mut self) -> Result> { + self.calls.push("joined".into()); + Ok(self.joined.clone()) + } + + fn facts(&mut self, room_id: &str) -> Result { + self.calls.push(format!("facts:{room_id}")); + if self.facts_fail.contains(room_id) { + bail!("the server did not answer for the state of {room_id}"); + } + match self.rooms.get(room_id) { + Some(room) => Ok(room.facts.clone()), + None => bail!("no such room"), + } + } + + fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result> { + self.calls.push(format!("events:{room_id}")); + let Some(room) = self.rooms.get(room_id) else { bail!("no such room") }; + let from = match after { + None => 0, + Some(after) => match room.events.iter().position(|event| event.event_id == after) { + Some(at) => at + 1, + None => return Ok(None), + }, + }; + let redacted: BTreeSet<&str> = room.events.iter().filter_map(|event| if let EventKind::Redaction { target } = &event.kind { Some(target.as_str()) } else { None }).collect(); + let served: Vec = room.events[from..] + .iter() + .map(|event| if self.prune && redacted.contains(event.event_id.as_str()) { RoomEvent { kind: EventKind::Other, content_keys: vec![], ..event.clone() } } else { event.clone() }) + .collect(); + let take = served.len().min(limit as usize); + Ok(Some(EventPage { events: served[..take].to_vec(), more: take < served.len() })) + } +} + +fn event(id: &str, sender: &str, ts_ms: u64, kind: EventKind) -> RoomEvent { + RoomEvent { event_id: id.into(), sender: sender.into(), ts_ms, content_keys: vec!["body".into(), "msgtype".into()], kind } +} + +fn said(id: &str, ts_ms: u64, body: &str) -> RoomEvent { + event(id, "@a:s", ts_ms, EventKind::Message { body: body.into() }) +} + +fn edit(id: &str, sender: &str, ts_ms: u64, target: &str, body: &str) -> RoomEvent { + event(id, sender, ts_ms, EventKind::Edit { target: target.into(), body: body.into() }) +} + +fn redaction(id: &str, ts_ms: u64, target: &str) -> RoomEvent { + event(id, "@a:s", ts_ms, EventKind::Redaction { target: target.into() }) +} + +struct Rig { + embedder: FakeEmbedder, + index: FakeSources, + estate: Estate, +} + +impl Rig { + fn new() -> Self { + let mut embedder = FakeEmbedder::new("d1"); + let index = FakeSources::default(); + let identity = crate::index::Embedder::model_identity(&mut embedder, Duration::from_secs(1)).unwrap(); + let meta = SourcesMeta { + managed_by: MANAGED_BY.into(), + corpus: CORPUS.into(), + model: identity.name, + model_digest: identity.digest, + dimensions: identity.dimensions, + chunker_version: chunk::CHUNKER_VERSION, + }; + index.plant(DescribedSources::Labelled { meta, cursors: BTreeMap::new() }); + Self { embedder, index, estate: estate(&[]) } + } + + fn run(&mut self, history: &mut FakeHistory) -> (Result, Vec) { + let mut journal = Vec::new(); + let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line)); + (out, journal) + } + + fn ok(&mut self, history: &mut FakeHistory) -> ChatRefreshed { + self.run(history).0.unwrap() + } + + fn points(&self) -> Vec { + self.index.state.lock().unwrap().chat.values().cloned().collect() + } + + fn ids(&self) -> BTreeSet { + self.index.state.lock().unwrap().chat.keys().cloned().collect() + } + + fn bodies(&self) -> Vec { + let mut bodies: Vec = self.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| message.body.clone())).collect(); + bodies.sort(); + bodies + } + + fn held_ids(&self, room: &str) -> Vec { + let mut ids: Vec = + self.points().iter().filter(|point| point.payload.room_id == room).flat_map(|point| point.payload.messages.iter().map(|message| message.event_id.clone())).collect(); + ids.sort(); + ids + } + + fn cursor(&self, room: &str) -> Option { + self.index.cursors().get(&cursor_key(room)).cloned() + } + + fn embedded(&self) -> Vec { + self.embedder.texts() + } + + fn seed(&mut self, room: &str, event_id: &str, ts: u64) { + let mut seeded = vec![ChatPoint { + id: chunk_id(room, event_id), + vector: vec![1.0; 4], + payload: chunk_payload_of(room, &[(event_id, ts)]), + }]; + self.index.upsert_chat(&seeded.split_off(0)).unwrap(); + let mut cursors = self.index.cursors(); + cursors.insert(cursor_key(room), format!("{}:{event_id}", chat_version())); + self.index.set_cursors(&cursors).unwrap(); + } +} + +fn chunk_payload_of(room: &str, messages: &[(&str, u64)]) -> ChatPayload { + let messages: Vec = messages.iter().map(|(id, ts)| ChatMessage { event_id: (*id).into(), sender: "@a:s".into(), ts_ms: *ts, body: format!("seeded {id}") }).collect(); + ChatPayload { room_id: room.into(), room_alias: None, text: "seeded".into(), messages } +} + +fn tombstone(rig: &Rig, room: &str) -> Option { + rig.index.cursors().get(&format!("{CHAT_PURGE_PREFIX}{room}")).cloned() +} + +// ---- scope: refusals first ---- + +#[test] +fn only_joined_rooms_are_looked_at_and_a_join_rule_is_not_the_test() { + let mut history = FakeHistory::default(); + history.room("!a:s", vec![said("$a1", 1, "in a")]).room("!b:s", vec![said("$b1", 2, "in b")]); + // A room the reader has not joined: scripted, but never listed. + history.rooms.insert("!c:s".into(), Scripted { facts: plain("!c:s"), events: vec![said("$c1", 3, "in c")] }); + let mut rig = Rig::new(); + let done = rig.ok(&mut history); + assert!(done.is_clean()); + assert!(!history.calls.iter().any(|call| call.contains("!c:s")), "{:?}", history.calls); + assert_eq!(rig.held_ids("!a:s"), ["$a1"]); + assert_eq!(rig.held_ids("!b:s"), ["$b1"]); + assert!(rig.held_ids("!c:s").is_empty() && done.rooms.iter().all(|report| report.room_id != "!c:s")); + let code = include_str!("room_scope.rs"); + assert!(!code[..code.find("#[cfg(test)]").unwrap()].contains(&["join", "_rule"].concat())); +} + +#[test] +fn a_dm_room_is_refused_unread_unindexed_and_purged() { + let mut history = FakeHistory::default(); + history.room("!dm:s", vec![said("$d1", 1, "private")]).facts("!dm:s", |facts| facts.direct = true); + let mut rig = Rig::new(); + rig.seed("!dm:s", "$old", 5); + let (done, journal) = rig.run(&mut history); + let done = done.unwrap(); + assert!(!history.called("events", "!dm:s")); + assert!(rig.points().is_empty() && rig.cursor("!dm:s").is_none() && rig.embedded().is_empty()); + assert_eq!(done.refused, [RefusedRoom { room_id: "!dm:s".into(), reason: Refusal::Direct, event_id: None }]); + assert!(!done.is_clean() && journal.iter().any(|line| line == "refused: !dm:s direct room"), "{journal:?}"); +} + +#[test] +fn each_estate_room_is_refused_by_id_and_by_alias_and_an_excluded_room_in_the_index_is_purged() { + let mut rig = Rig::new(); + rig.estate = estate(&["!byid:s"]); + let mut history = FakeHistory::default(); + history.room("!byid:s", vec![said("$i1", 1, "x")]); + for (n, alias) in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:GAMECULT.ORG".to_owned()]).enumerate() { + history.room(&format!("!al{n}:s"), vec![said("$x", 1, "x")]).facts(&format!("!al{n}:s"), |facts| facts.alias = Some(alias)); + } + history.room("!alt:s", vec![said("$x", 1, "x")]).facts("!alt:s", |facts| facts.alt_aliases = vec!["#other:s".into(), "#patrons:gamecult.org".into()]); + history.room("!ok:s", vec![said("$ok1", 1, "fine")]); + history.room("!guide:s", vec![said("$g1", 1, "fine")]).facts("!guide:s", |facts| facts.alias = Some("#users-guide:gamecult.org".into())); + history.room("!club:s", vec![said("$c1", 1, "fine")]).facts("!club:s", |facts| facts.alias = Some("#patrons-club:gamecult.org".into())); + for room in ["!byid:s", "!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] { + rig.seed(room, "$old", 5); + } + let mut cursors = rig.index.cursors(); + cursors.insert(format!("{CHAT_PURGE_PREFIX}!al0:s"), "1..2".into()); + rig.index.set_cursors(&cursors).unwrap(); + let done = rig.ok(&mut history); + assert!(!history.called("facts", "!byid:s") && !history.called("events", "!byid:s")); + for room in ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] { + assert!(history.called("facts", room) && !history.called("events", room), "{room}"); + } + let refused: BTreeSet<&str> = done.refused.iter().map(|refused| refused.room_id.as_str()).collect(); + assert_eq!(refused, ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s", "!byid:s"].into()); + assert!(done.refused.iter().all(|refused| refused.reason == Refusal::Estate) && !done.is_clean()); + let held: BTreeSet = rig.points().iter().map(|point| point.payload.room_id.clone()).collect(); + assert_eq!(held, ["!guide:s".to_owned(), "!club:s".to_owned(), "!ok:s".to_owned()].into(), "lookalikes and plain rooms are in; every estate room is out"); + assert!(rig.index.cursors().keys().all(|key| !key.starts_with("chat:!al") && !key.starts_with("chat:!byid") && !key.starts_with("chat:!alt"))); + assert_eq!(tombstone(&rig, "!al0:s").as_deref(), Some("1..2"), "a tombstone is never touched by a refusal"); +} + +#[test] +fn an_encrypted_room_is_refused_and_an_encrypted_event_in_a_plain_room_is_never_stored() { + let mut history = FakeHistory::default(); + history.room("!enc:s", vec![said("$e1", 1, "x")]).facts("!enc:s", |facts| facts.encrypted = true); + history.room("!plain:s", vec![said("$p1", 1, "visible"), event("$p2", "@a:s", 2, EventKind::Other), said("$p3", 3, "also visible")]); + let mut rig = Rig::new(); + let done = rig.ok(&mut history); + assert_eq!(done.refused[0].reason, Refusal::Encrypted); + assert!(!history.called("events", "!enc:s")); + assert_eq!(rig.held_ids("!plain:s"), ["$p1", "$p3"]); + assert_eq!(rig.cursor("!plain:s").unwrap(), format!("{}:$p3", chat_version()), "the cursor passes an Other event"); +} + +#[test] +fn a_bridged_room_is_refused_whole_before_any_event_is_read_and_its_chunks_are_purged() { + let mut history = FakeHistory::default(); + for (n, state_type) in ["m.bridge", "uk.half-shot.bridge"].into_iter().enumerate() { + let room = format!("!br{n}:s"); + history.room(&room, vec![said("$x", 1, "x")]).facts(&room, |facts| facts.bridge = Some(state_type.into())); + } + let mut rig = Rig::new(); + rig.seed("!br0:s", "$old", 5); + let done = rig.ok(&mut history); + assert_eq!(done.refused.len(), 2); + assert!(done.refused.iter().all(|refused| matches!(refused.reason, Refusal::Bridged(_)))); + assert!(!history.calls.iter().any(|call| call.starts_with("events:"))); + assert!(rig.points().is_empty() && rig.index.cursors().is_empty()); +} + +#[test] +fn a_bridged_platforms_sender_or_content_refuses_the_room_before_anything_is_embedded_and_purges_what_it_held() { + let cases = [ + event("$bad", "@discord_123:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }), + event("$bad", "@_discord_9:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }), + event("$bad", "@discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }), + event("$bad", "@mautrix_discord:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }), + event("$bad", "@_bridge_discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }), + RoomEvent { content_keys: vec!["body".into(), "fi.mau.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") }, + RoomEvent { content_keys: vec!["uk.half-shot.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") }, + ]; + for bad in cases { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$ok", 1, "before"), bad]); + history.room("!other:s", vec![said("$o1", 1, "other room")]); + let mut rig = Rig::new(); + rig.seed("!r:s", "$prev", 0); + let (done, journal) = rig.run(&mut history); + let done = done.unwrap(); + assert!(!rig.embedded().iter().any(|text| text.contains("SECRET-BODY-TEXT") || text.contains("before")), "nothing of the page is embedded"); + assert!(rig.held_ids("!r:s").is_empty() && rig.cursor("!r:s").is_none()); + assert_eq!(rig.held_ids("!other:s"), ["$o1"]); + assert_eq!(done.refused.len(), 1); + assert_eq!((done.refused[0].room_id.as_str(), done.refused[0].event_id.as_deref()), ("!r:s", Some("$bad"))); + assert!(!done.is_clean()); + assert!(journal.iter().all(|line| !line.contains("SECRET-BODY-TEXT")), "{journal:?}"); + } +} + +#[test] +fn a_lookalike_of_the_bridged_platform_is_kept() { + let mut history = FakeHistory::default(); + let kept = ["@discourse:s", "@disco:s", "@eureka:s"].map(|sender| event(&format!("${}", &sender[1..6]), sender, 1, EventKind::Message { body: "hi".into() })); + let mut keyed = said("$k", 2, "hi"); + keyed.content_keys.push("discourse".into()); + let mut events = kept.to_vec(); + events.push(keyed); + history.room("!r:s", events); + let mut rig = Rig::new(); + assert!(rig.ok(&mut history).is_clean()); + assert_eq!(rig.held_ids("!r:s").len(), 4); +} + +#[test] +fn the_collection_is_never_created_by_chat() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 1, "x")]); + for described in [None, Some(DescribedSources::Absent), Some(DescribedSources::Unlabelled)] { + let mut rig = Rig::new(); + match described { + None => rig.index.state.lock().unwrap().described = None, + Some(described) => rig.index.plant(described), + } + let error = format!("{:#}", rig.run(&mut history).0.unwrap_err()); + assert!(error.contains("forge refresh creates it"), "{error}"); + assert_eq!(rig.index.state.lock().unwrap().recreated, 0); + assert!(rig.points().is_empty() && rig.embedded().is_empty()); + } +} + +// ---- redactions, edits, replays ---- + +#[test] +fn a_redacted_event_is_absent_after_the_next_refresh_and_a_first_or_only_message_changes_or_drops_its_chunk() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), said("$c", 30, "gamma")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + let first_id = chunk_id("!r:s", "$a"); + assert_eq!(rig.ids(), [first_id.clone()].into()); + history.push("!r:s", vec![redaction("$r1", 40, "$b")]); + rig.ok(&mut history); + assert_eq!(rig.held_ids("!r:s"), ["$a", "$c"]); + assert!(!rig.points()[0].payload.text.contains("beta")); + history.push("!r:s", vec![redaction("$r2", 50, "$a")]); + rig.ok(&mut history); + assert_eq!(rig.ids(), [chunk_id("!r:s", "$c")].into(), "the first message left, so the id changed and the old one is gone"); + history.push("!r:s", vec![redaction("$r3", 60, "$c")]); + rig.ok(&mut history); + assert!(rig.points().is_empty(), "a chunk the redaction empties is deleted"); +} + +#[test] +fn a_redaction_in_the_same_page_as_its_message_never_reaches_the_embedder() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 10, "keep me"), said("$gone", 20, "REDACTED-BODY"), redaction("$r", 30, "$gone")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + assert!(rig.embedded().iter().all(|text| !text.contains("REDACTED-BODY"))); + assert_eq!(rig.held_ids("!r:s"), ["$a"]); +} + +#[test] +fn an_edit_replaces_the_body_in_place_and_a_stranger_an_unknown_target_or_an_edit_of_an_edit_change_nothing() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 10, "old words"), said("$b", 20, "other")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + let before = rig.points(); + history.push("!r:s", vec![edit("$e1", "@a:s", 30, "$a", "new words")]); + rig.ok(&mut history); + let after = rig.points(); + assert_eq!(after.len(), 1); + let first = &after[0].payload.messages[0]; + assert_eq!((first.event_id.as_str(), first.sender.as_str(), first.ts_ms, first.body.as_str()), ("$a", "@a:s", 10, "new words")); + assert_eq!(after[0].id, before[0].id); + assert!(after[0].payload.text.contains("new words") && !after[0].payload.text.contains("old words")); + assert!(rig.embedded().last().unwrap().contains("new words")); + let calls = rig.embedder.state.lock().unwrap().embedded.len(); + history.push("!r:s", vec![edit("$e2", "@stranger:s", 40, "$a", "hijack"), edit("$e3", "@a:s", 41, "$nothing", "lost"), edit("$e4", "@a:s", 42, "$e1", "edit of an edit")]); + rig.ok(&mut history); + assert_eq!(rig.points(), after); + assert_eq!(rig.embedder.state.lock().unwrap().embedded.len(), calls, "no embed call for edits that change nothing"); + assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$e4", chat_version()), "the cursor is the last event of any kind"); +} + +#[test] +fn replay_after_a_crash_between_the_upsert_and_the_cursor_neither_duplicates_nor_loses() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), edit("$e", "@a:s", 25, "$a", "alpha two"), said("$c", 30, "gamma")]); + let mut rig = Rig::new(); + // One cursor write (the empty start) succeeds, then the page's upsert is acknowledged and the next write fails. + rig.index.state.lock().unwrap().fail_cursor_writes_after = Some(1); + assert!(!rig.run(&mut history).0.unwrap().failed.is_empty()); + let (ids, held) = (rig.ids(), rig.held_ids("!r:s")); + assert_eq!(held, ["$a", "$b", "$c"]); + rig.index.state.lock().unwrap().fail_cursor_writes_after = None; + let done = rig.ok(&mut history); + assert!(done.is_clean()); + assert_eq!((rig.ids(), rig.held_ids("!r:s")), (ids, held)); + assert!(rig.bodies().contains(&"alpha two".to_owned())); + assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$c", chat_version())); +} + +#[test] +fn a_failed_upsert_leaves_the_cursor_and_a_failed_embed_or_delete_deletes_nothing_the_index_held() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + let (cursor, points) = (rig.cursor("!r:s"), rig.points()); + history.push("!r:s", vec![said("$c", 30, "gamma")]); + rig.index.state.lock().unwrap().fail_chat_upserts = true; + assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]); + assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone())); + rig.index.state.lock().unwrap().fail_chat_upserts = false; + rig.embedder.state.lock().unwrap().embed_fails = true; + assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]); + assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone())); + rig.embedder.state.lock().unwrap().embed_fails = false; + // A redaction of the first message writes a new chunk, then fails to delete the old one: the cursor stays. + history.push("!r:s", vec![redaction("$r", 40, "$a")]); + rig.index.state.lock().unwrap().fail_chat_deletes = true; + assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]); + assert_eq!(rig.cursor("!r:s"), cursor); + rig.index.state.lock().unwrap().fail_chat_deletes = false; + assert!(rig.ok(&mut history).is_clean()); + assert_eq!(rig.held_ids("!r:s"), ["$b", "$c"]); +} + +// ---- chunking through the job ---- + +#[test] +fn appending_to_the_tail_over_many_runs_equals_one_run_over_all() { + let all: Vec = (0..80).map(|n| said(&format!("$m{n:03}"), 1_000 * n + if n >= 40 { 5_000_000 } else { 0 }, &format!("message number {n} {}", "x".repeat((n as usize * 37) % 400)))).collect(); + let mut whole_history = FakeHistory::default(); + whole_history.room("!r:s", all.clone()); + let mut whole = Rig::new(); + whole.ok(&mut whole_history); + let mut grown_history = FakeHistory::default(); + grown_history.room("!r:s", vec![]); + let mut grown = Rig::new(); + for batch in all.chunks(7) { + grown_history.push("!r:s", batch.to_vec()); + grown.ok(&mut grown_history); + } + assert!(whole.points().len() > 2); + let shape = |rig: &Rig| rig.points().into_iter().map(|point| (point.id, point.payload)).collect::>(); + assert_eq!(shape(&grown), shape(&whole)); +} + +#[test] +fn a_long_message_is_cut_with_a_marker_and_every_point_satisfies_the_payload_contract() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 10, &"é".repeat(CHAT_BODY_CHARS_MAX * 2)), said("$b", 20, "short"), said("$c", 99_000_000, "later")]); + history.facts("!r:s", |facts| { + facts.alias = Some("#swarm:gamecult.org".into()); + facts.name = Some("Swarm".into()); + }); + let mut rig = Rig::new(); + rig.ok(&mut history); + let points = rig.points(); + assert_eq!(points.len(), 2); + for point in &points { + let payload = &point.payload; + assert_eq!((payload.room_id.as_str(), payload.room_alias.as_deref()), ("!r:s", Some("#swarm:gamecult.org"))); + assert_eq!(point.id, chunk_id("!r:s", &payload.messages[0].event_id)); + assert_eq!(payload.text, super::chat_chunk::embedded_text("Swarm", &payload.messages)); + assert!(!payload.messages.is_empty()); + } + let long = &points.iter().find(|point| point.payload.messages[0].event_id == "$a").unwrap().payload.messages[0]; + assert_eq!(long.body.chars().count(), CHAT_BODY_CHARS_MAX); + assert!(long.body.ends_with("[cut]")); +} + +#[test] +fn a_lost_cursor_event_or_a_version_bump_re_ingests_that_room_alone() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]).room("!q:s", vec![said("$q1", 10, "quiet")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + let q_ids: BTreeSet = rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect(); + let recreated = rig.index.state.lock().unwrap().recreated; + for bump in [Some("0.0".to_owned()), None] { + let mut cursors = rig.index.cursors(); + let value = match &bump { + Some(old) => format!("{old}:$b"), + None => format!("{}:$gone-from-the-server", chat_version()), + }; + cursors.insert(cursor_key("!r:s"), value); + rig.index.set_cursors(&cursors).unwrap(); + let before = rig.embedded().len(); + assert!(rig.ok(&mut history).is_clean()); + assert_eq!(rig.held_ids("!r:s"), ["$a", "$b"]); + assert!(rig.embedded().len() > before, "the room was re-embedded"); + assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect::>(), q_ids); + assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$b", chat_version())); + } + assert_eq!(rig.index.state.lock().unwrap().recreated, recreated); +} + +#[test] +fn retire_chat_removes_rooms_no_longer_kept_and_given_none_all_chat_and_touches_nothing_else() { + let mut rig = Rig::new(); + rig.index.upsert_sources(&[SourcePoint { + id: "forge-1".into(), + vector: vec![1.0; 4], + payload: SourcePayload { repo: "Repo".into(), path: "a".into(), commit: "c".into(), blob: "b".into(), line_start: 1, line_end: 1, chunk_index: 0, ext: "rs".into(), text: "t".into() }, + }]) + .unwrap(); + for room in ["!a:s", "!b:s"] { + rig.seed(room, "$e", 1); + } + let mut cursors = rig.index.cursors(); + cursors.insert("Repo".into(), "c".into()); + cursors.insert(format!("{CHAT_PURGE_PREFIX}!a:s"), "1..2".into()); + rig.index.set_cursors(&cursors).unwrap(); + let retired = retire_chat(&mut rig.index, &["!a:s".to_owned()].into()).unwrap(); + assert_eq!(retired.retired, ["!b:s"]); + assert_eq!(rig.held_ids("!a:s"), ["$e"]); + assert!(rig.held_ids("!b:s").is_empty() && rig.cursor("!b:s").is_none()); + assert_eq!(retire_chat(&mut rig.index, &BTreeSet::new()).unwrap().retired, ["!a:s"]); + assert!(rig.points().is_empty()); + assert_eq!(rig.index.state.lock().unwrap().points.len(), 1, "a forge point stays"); + let kept: Vec = rig.index.cursors().into_keys().collect(); + assert_eq!(kept, ["Repo".to_owned(), format!("{CHAT_PURGE_PREFIX}!a:s")]); + // A room the reader left is retired by the refresh itself. + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 1, "x")]); + rig.seed("!gone:s", "$e", 1); + rig.ok(&mut history); + assert!(rig.held_ids("!gone:s").is_empty()); +} + +// ---- the credential scrub through the job ---- + +fn planted() -> Vec<(ScrubKind, String, String)> { + let t = |kind, secret: String, needle: &str| (kind, secret, needle.to_owned()); + vec![ + t(ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2a1B2"), + t(ScrubKind::Bearer, format!("Bearer {}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"), + t(ScrubKind::Bearer, format!("bearer:{}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"), + t(ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3q7W3"), + t(ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5m4N5"), + t(ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"), + t(ScrubKind::KeyShape, format!("xxx-ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"), + t(ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2C3"), + t(ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7r6T7"), + t(ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"), + t(ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"), + t(ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"), + t(ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"), + t(ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"), + t(ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF"), + t(ScrubKind::KeyShape, "xAKIA0123456789ABCDEF".into(), "0123456789ABCDEF"), + t(ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_"), + t(ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR"), + t(ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5"), + t(ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz"), + t(ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654"), + t(ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ"), + t(ScrubKind::PemBlock, "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfake\n-----END RSA PRIVATE KEY-----".into(), "MIIEowIBAAKCAQEAfake"), + t(ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake"), + t(ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake"), + t(ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2"), + t(ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345"), + ] +} + +#[test] +fn planted_credentials_never_reach_the_chunk_the_embedder_or_the_journal_in_a_message_or_an_edit() { + for (kind, secret, needle) in planted() { + for shape in ["see {} now", "see {}", "{}\nnext line"] { + let (dirty, clean) = (shape.replace("{}", &secret), "nothing here".to_owned()); + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$m", 10, &dirty), said("$e", 20, &clean), edit("$ed", "@a:s", 30, "$e", &dirty)]); + let mut rig = Rig::new(); + let (done, journal) = rig.run(&mut history); + let done = done.unwrap(); + assert!(done.is_clean(), "a scrub hit does not make a run unclean"); + let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::>()); + for seen in [stored, rig.embedded().join("\n"), journal.join("\n")] { + assert!(!seen.contains(&needle), "{kind}: {secret:?} leaked into {seen}"); + } + assert!(rig.bodies().iter().filter(|body| body.contains("")).count() >= 2, "{kind}: both bodies show the replacement"); + let hits: Vec<(&str, ScrubKind)> = done.scrubbed.iter().map(|hit| (hit.event_id.as_str(), hit.kind)).collect(); + assert!(hits.contains(&("$m", kind)) && hits.contains(&("$ed", kind)), "{kind}: {hits:?}"); + assert!(done.scrubbed.iter().all(|hit| hit.room_id == "!r:s")); + assert!(journal.contains(&format!("scrubbed: !r:s $m {kind}")), "{journal:?}"); + } + } +} + +#[test] +fn the_scrub_meets_edits_redactions_the_body_cut_replays_and_version_bumps() { + let token = format!("ghp_{}", "k2L9".repeat(9)); + let mut history = FakeHistory::default(); + history.room( + "!r:s", + vec![ + said("$clean", 10, "all clean"), + said("$dirty", 20, &format!("token {token} inside")), + edit("$adds", "@a:s", 30, "$clean", &format!("now with {token}")), + edit("$removes", "@a:s", 40, "$dirty", "token removed"), + said("$gone", 50, &format!("secret {token}")), + redaction("$r", 60, "$gone"), + said("$straddle", 70, &format!("{} {token} {}", "x".repeat(CHAT_BODY_CHARS_MAX - 10), "y".repeat(100))), + ], + ); + let mut rig = Rig::new(); + let done = rig.ok(&mut history); + assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9"))); + assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9"))); + let by_id: BTreeMap = rig.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| (message.event_id.clone(), message.body.clone()))).collect(); + assert_eq!(by_id["$clean"], "now with "); + assert_eq!(by_id["$dirty"], "token removed"); + assert!(!by_id.contains_key("$gone")); + assert!(by_id["$straddle"].chars().count() <= CHAT_BODY_CHARS_MAX && !by_id["$straddle"].contains("ghp_")); + assert!(done.scrubbed.iter().any(|hit| hit.event_id == "$adds") && done.scrubbed.iter().any(|hit| hit.event_id == "$gone")); + let (ids, texts) = (rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::>()); + // Replay from the start, as after a lost cursor: identical ids and text. + let mut cursors = rig.index.cursors(); + cursors.remove(&cursor_key("!r:s")); + rig.index.set_cursors(&cursors).unwrap(); + rig.ok(&mut history); + assert_eq!((rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::>()), (ids, texts)); + // An older scrub version re-ingests the room and scrubs it again from the history. + let mut cursors = rig.index.cursors(); + cursors.insert(cursor_key("!r:s"), format!("{}.0:$straddle", huginn_mind::redact::SCRUB_VERSION - 1)); + rig.index.set_cursors(&cursors).unwrap(); + rig.ok(&mut history); + assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9"))); +} + +#[test] +fn ordinary_text_is_stored_byte_for_byte() { + let mut history = FakeHistory::default(); + let same = "risk-averse task-force sk- 3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c !abc:s user@example.com ssh://git@host/r.git the bearer of bad news"; + history.room("!r:s", vec![said("$a", 1, same)]); + let mut rig = Rig::new(); + let done = rig.ok(&mut history); + assert_eq!(rig.bodies(), [same]); + assert!(done.scrubbed.is_empty()); +} + +// ---- steward purge ---- + +fn window(from_ms: u64, to_ms: u64) -> PurgeWindow { + PurgeWindow { from_ms, to_ms } +} + +fn purge_history() -> FakeHistory { + let mut history = FakeHistory::default(); + // t1 = 1000, t2 = 3000: $b at t1 and $c inside are purged, $d at t2 is kept. + history.room("!r:s", vec![said("$a", 500, "before"), said("$b", 1000, "PURGED-B"), said("$c", 2000, "PURGED-C"), said("$d", 3000, "after"), said("$e", 4000, "later")]); + history.room("!other:s", vec![said("$o1", 1000, "same time other room"), said("$o2", 2000, "same time")]); + history +} + +#[test] +fn a_purge_window_removes_exactly_its_events_and_re_chunks_the_neighbours() { + let mut history = purge_history(); + let mut rig = Rig::new(); + rig.ok(&mut history); + let other_ids: BTreeSet = rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect(); + assert_eq!(rig.held_ids("!r:s"), ["$a", "$b", "$c", "$d", "$e"]); + assert!(request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap()); + let embedded_before = rig.embedded().len(); + rig.ok(&mut history); + assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$e"], "t1 is inside, t2 is not"); + assert!(rig.embedded()[embedded_before..].iter().all(|text| !text.contains("PURGED"))); + assert!(rig.points().iter().all(|point| !point.payload.text.contains("PURGED"))); + assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect::>(), other_ids); + assert_eq!(rig.held_ids("!other:s"), ["$o1", "$o2"]); + assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("1000..3000")); +} + +#[test] +fn a_purged_first_message_changes_the_chunk_id_and_an_emptied_chunk_is_deleted() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 1000, "x"), said("$b", 2000, "y"), said("$c", 99_000_000, "far")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + assert_eq!(rig.ids().len(), 2); + request_purge(&mut rig.index, "!r:s", window(900, 1500)).unwrap(); + rig.ok(&mut history); + assert_eq!(rig.ids(), [chunk_id("!r:s", "$b"), chunk_id("!r:s", "$c")].into()); + request_purge(&mut rig.index, "!r:s", window(99_000_000, 99_000_001)).unwrap(); + rig.ok(&mut history); + assert_eq!(rig.held_ids("!r:s"), ["$b"]); +} + +#[test] +fn a_purged_window_does_not_come_back() { + let mut history = purge_history(); + let mut rig = Rig::new(); + rig.ok(&mut history); + request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap(); + rig.ok(&mut history); + let want = ["$a", "$d", "$e"]; + // New events, an in-window edit of an outside message, a redaction. + history.push("!r:s", vec![said("$f", 5000, "new"), edit("$ed", "@a:s", 2500, "$a", "EDIT-IN-WINDOW"), redaction("$rd", 6000, "$e")]); + rig.ok(&mut history); + assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"]); + assert!(rig.bodies().contains(&"before".to_owned()) && !rig.bodies().iter().any(|body| body.contains("EDIT-IN-WINDOW")), "the in-window edit is dropped and the original stays"); + // A lost cursor re-ingests from the start; a version bump re-ingests too. + for cursor in [None, Some("0.0:$f".to_owned())] { + let mut cursors = rig.index.cursors(); + match cursor { + None => cursors.remove(&cursor_key("!r:s")), + Some(old) => cursors.insert(cursor_key("!r:s"), old), + }; + rig.index.set_cursors(&cursors).unwrap(); + rig.ok(&mut history); + assert!(rig.bodies().iter().all(|body| !body.contains("PURGED")), "{:?}", rig.held_ids("!r:s")); + assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"], "$e stays redacted by the history; {want:?} before the new events"); + } + // The reader leaves the room and joins again. + let mut left = FakeHistory::default(); + left.room("!other:s", vec![said("$o1", 1000, "x")]); + rig.ok(&mut left); + assert!(rig.held_ids("!r:s").is_empty() && tombstone(&rig, "!r:s").is_some()); + rig.ok(&mut history); + assert!(rig.bodies().iter().all(|body| !body.contains("PURGED"))); +} + +#[test] +fn a_purge_of_an_unindexed_window_writes_only_the_tombstone_and_still_holds() { + let mut rig = Rig::new(); + let writes = rig.index.state.lock().unwrap().cursor_writes.len(); + assert!(request_purge(&mut rig.index, "!never:s", window(10, 20)).unwrap()); + assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes + 1); + assert!(rig.points().is_empty() && rig.embedded().is_empty()); + let mut history = FakeHistory::default(); + history.room("!never:s", vec![said("$a", 5, "ok"), said("$b", 15, "PURGED"), said("$c", 25, "ok")]); + rig.ok(&mut history); + assert_eq!(rig.held_ids("!never:s"), ["$a", "$c"]); + assert!(rig.embedded().iter().all(|text| !text.contains("PURGED"))); +} + +#[test] +fn a_repeated_or_overlapping_request_changes_nothing_or_merges_and_the_33rd_window_is_refused() { + let mut rig = Rig::new(); + assert!(request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap()); + let writes = rig.index.state.lock().unwrap().cursor_writes.len(); + assert!(!request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap()); + assert!(!request_purge(&mut rig.index, "!r:s", window(12, 18)).unwrap()); + assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes, "no write for a request the tombstone already holds"); + assert!(request_purge(&mut rig.index, "!r:s", window(15, 30)).unwrap()); + assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("10..30")); + for n in 0..(PURGE_WINDOWS_MAX as u64 - 1) { + request_purge(&mut rig.index, "!r:s", window(100 + n * 10, 105 + n * 10)).unwrap(); + } + let error = format!("{:#}", request_purge(&mut rig.index, "!r:s", window(10_000, 10_001)).unwrap_err()); + assert!(error.contains("32"), "{error}"); + assert!(request_purge(&mut Rig::new().index, "!r:s", window(5, 5)).is_err()); +} + +// ---- the source stays in its lane ---- + +#[test] +fn the_ingest_code_is_pure_over_its_ports_and_names_no_credential_shape_no_second_literal_and_no_join_rule() { + let sources = [("chat.rs", include_str!("chat.rs")), ("chat_chunk.rs", include_str!("chat_chunk.rs")), ("chat_purge.rs", include_str!("chat_purge.rs")), ("room_scope.rs", include_str!("room_scope.rs"))]; + let process = ["Command", "::new"].concat(); + for (name, source) in sources { + let code = source.split("#[cfg(test)]").next().unwrap(); + assert!(!code.contains(&process) && !code.contains("std::process"), "{name}"); + assert!(!code.to_ascii_lowercase().contains("join_rule"), "{name}"); + if name != "room_scope.rs" { + assert!(!code.to_ascii_lowercase().contains("discord"), "{name}"); + } + for shape in ["sk-", "sk_live_", "ghp_", "github_pat_", "glpat-", "AKIA", "xoxb-", "PRIVATE KEY", "Bearer"] { + assert!(!code.contains(shape), "{name} names {shape}"); + } + let defined = usize::from(name == "chat.rs"); + assert_eq!((code.matches("\"chat:").count(), code.matches("\"chat-purge:").count()), (defined, defined), "{name}: each prefix is defined once"); + } + let chat = include_str!("chat.rs"); + assert_eq!(chat.matches("scrub(").count(), 2, "scrub is called for a body and for the room name, nowhere else"); +} + +// ---- redacted edits, redacted originals, late edits ---- + +const EDIT_SECRET: &str = "EDIT-SECRET"; + +fn nothing_secret_stored(rig: &Rig) { + let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::>()); + assert!(!stored.contains(EDIT_SECRET), "{stored}"); +} + +#[test] +fn a_redacted_edit_in_the_same_page_reverts_the_message_to_its_earlier_body() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$m", 10, "original words"), edit("$e1", "@a:s", 20, "$m", "first edit"), edit("$e2", "@a:s", 30, "$m", EDIT_SECRET), redaction("$r", 40, "$e2")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + assert_eq!(rig.bodies(), ["first edit"]); + assert!(rig.embedded().iter().all(|text| !text.contains(EDIT_SECRET))); + history.room("!q:s", vec![said("$m", 10, "orig"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET), redaction("$r", 30, "$e")]); + rig.ok(&mut history); + assert!(rig.bodies().contains(&"orig".to_owned())); + nothing_secret_stored(&rig); +} + +#[test] +fn a_redaction_of_an_edit_a_later_run_stored_rebuilds_the_room_and_the_secret_does_not_return() { + let mut history = FakeHistory { prune: true, ..FakeHistory::default() }; + history.room("!r:s", vec![said("$m", 10, "original"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET)]); + history.room("!q:s", vec![said("$q", 10, "quiet")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + assert!(rig.bodies().iter().any(|body| body == EDIT_SECRET), "the edit applied while it stood"); + let recreated = rig.index.state.lock().unwrap().recreated; + history.push("!r:s", vec![redaction("$r", 30, "$e")]); + let done = rig.ok(&mut history); + assert!(done.is_clean()); + nothing_secret_stored(&rig); + assert!(rig.bodies().contains(&"original".to_owned())); + assert_eq!(rig.held_ids("!q:s"), ["$q"]); + assert_eq!(rig.index.state.lock().unwrap().recreated, recreated); + assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$r", chat_version())); + // The rebuild does not loop: the redaction it replays is not a trigger the second time, and nothing changes. + let ids = rig.ids(); + rig.ok(&mut history); + assert_eq!(rig.ids(), ids); +} + +#[test] +fn a_redacted_original_takes_its_live_and_late_edits_with_it_in_a_page_across_runs_and_across_a_rebuild() { + let script = |history: &mut FakeHistory| { + history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", "live edit"), redaction("$re", 25, "$e1"), redaction("$rm", 30, "$m"), edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]); + history.prune = true; + }; + // One page. + let (mut history, mut rig) = (FakeHistory::default(), Rig::new()); + script(&mut history); + rig.ok(&mut history); + nothing_secret_stored(&rig); + assert!(rig.points().is_empty()); + // Across runs: the original and a live edit stand, then it is redacted, then a late edit arrives. + let (mut history, mut rig) = (FakeHistory { prune: true, ..FakeHistory::default() }, Rig::new()); + history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", EDIT_SECRET)]); + rig.ok(&mut history); + history.push("!r:s", vec![redaction("$rm", 30, "$m")]); + rig.ok(&mut history); + nothing_secret_stored(&rig); + history.push("!r:s", vec![edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]); + rig.ok(&mut history); + nothing_secret_stored(&rig); + assert!(rig.points().is_empty()); + // Across a rebuild: a lost cursor re-reads the whole history. + let mut cursors = rig.index.cursors(); + cursors.remove(&cursor_key("!r:s")); + rig.index.set_cursors(&cursors).unwrap(); + rig.ok(&mut history); + nothing_secret_stored(&rig); + assert!(rig.points().is_empty()); +} + +#[test] +fn a_stranger_cannot_edit_a_message_of_the_same_page() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$m", 10, "mine"), edit("$e", "@stranger:s", 20, "$m", "hijacked")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + assert_eq!(rig.bodies(), ["mine"]); +} + +#[test] +fn the_room_name_is_scrubbed_before_it_is_embedded_or_stored() { + let mut history = FakeHistory::default(); + let token = format!("ghp_{}", "k2L9".repeat(9)); + history.room("!r:s", vec![said("$m", 10, "hello")]).facts("!r:s", |facts| facts.name = Some(format!("Swarm {token}"))); + let mut rig = Rig::new(); + let (done, journal) = rig.run(&mut history); + let done = done.unwrap(); + assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9"))); + assert!(rig.points().iter().all(|point| !point.payload.text.contains("k2L9k2L9") && point.payload.text.starts_with("Swarm "))); + assert!(done.scrubbed.iter().any(|hit| hit.event_id == "(name)") && journal.iter().all(|line| !line.contains("k2L9k2L9"))); +} + +#[test] +fn a_token_split_by_zero_width_characters_leaves_no_tail_in_a_stored_body() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$m", 10, &format!("key sk-{}\u{200b}{} done", "q7W3".repeat(4), "q7W3".repeat(4)))]); + let mut rig = Rig::new(); + rig.ok(&mut history); + assert_eq!(rig.bodies(), ["key done"]); +} + +#[test] +fn retire_goes_on_past_a_failed_delete_and_names_each_failure() { + let mut rig = Rig::new(); + for room in ["!a:s", "!b:s", "!c:s"] { + rig.seed(room, "$e", 1); + } + rig.index.state.lock().unwrap().fail_chat_deletes_for = ["!a:s".to_owned()].into(); + let done = retire_chat(&mut rig.index, &BTreeSet::new()).unwrap(); + assert_eq!(done.retired, ["!b:s", "!c:s"]); + assert_eq!(done.failed.len(), 1); + assert_eq!(done.failed[0].0, "!a:s"); + assert_eq!(rig.held_ids("!a:s"), ["$e"]); + assert!(rig.cursor("!a:s").is_some(), "the cursor stays so the next run finds it again"); + // The refresh reports it too. + let mut history = FakeHistory::default(); + history.room("!z:s", vec![said("$z", 1, "x")]); + let done = rig.ok(&mut history); + assert!(done.failed.contains(&"!a:s".to_owned()) && !done.is_clean()); +} + +#[test] +fn a_facts_error_on_a_joined_room_keeps_its_points_and_cursor_and_is_reported() { + let mut history = FakeHistory::default(); + history.room("!r:s", vec![said("$a", 1, "x")]); + let mut rig = Rig::new(); + rig.ok(&mut history); + let (cursor, points) = (rig.cursor("!r:s"), rig.points()); + history.facts_fail.insert("!r:s".into()); + let (done, journal) = rig.run(&mut history); + let done = done.unwrap(); + assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor, points)); + assert_eq!(done.failed, ["!r:s"]); + assert!(done.rooms.iter().any(|report| report.room_id == "!r:s" && report.failure.is_some())); + assert!(journal.iter().any(|line| line.contains("!r:s") && line.contains("failed"))); +} + +#[test] +fn the_tail_on_equal_ends_is_the_greatest_chunk_id() { + let mut rig = Rig::new(); + let (a, b) = (chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b")); + rig.index.upsert_chat(&[ + ChatPoint { id: a.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$a", 5)]) }, + ChatPoint { id: b.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$b", 5)]) }, + ]) + .unwrap(); + assert_eq!(rig.index.chat_tail("!r:s").unwrap().unwrap().id, a.max(b)); +} diff --git a/crates/huginn-daemon/src/sources/mod.rs b/crates/huginn-daemon/src/sources/mod.rs index 963296b..5e4133d 100644 --- a/crates/huginn-daemon/src/sources/mod.rs +++ b/crates/huginn-daemon/src/sources/mod.rs @@ -3,26 +3,37 @@ //! for semantic search. Each point's `corpus` payload (`forge` or `chat`) keeps //! the two apart; a forge search filters `forge` and a chat search `chat`. //! -//! Two truths, one writer. The forge is the truth of which repos exist and what -//! they hold; the collection is derived from it and can be deleted whenever. -//! `refresh` is the only writer of the collection, run by the `huginn-sources` -//! oneshot job: first build, incremental refresh, rebuild after a model or -//! chunker change, and retirement are all the one routine (delete by filter, -//! ingest at a commit, rewrite the cursors). The daemon only reads: `search` -//! is its whole part, and nothing in `serve` or the worker calls a write. +//! Two truths, one job. The forge is the truth of which repos exist and what +//! they hold, Synapse of what the Minds' rooms said; the collection is derived +//! from them and can be deleted whenever (but for a steward's purge tombstones, see +//! `chat_purge`). `refresh` is the forge's writer and the only creator of the +//! collection, run by the `huginn-sources` oneshot job: first build, incremental +//! refresh, rebuild after a model or chunker change, and retirement are all the one +//! routine (delete by filter, ingest at a commit, rewrite the cursors). `chat::refresh_chat` +//! joins it in the same job and the same cursors object, as the only writer of chat points; +//! it requires the collection to exist and never recreates it. The daemon only reads: `search` +//! and `search_chat` are its whole part, and nothing in `serve` or the worker calls a write. //! -//! A cursor is the commit whose chunks the collection holds for a repo. It -//! lives in the collection's own metadata, so it is lost with the collection +//! A cursor is the commit (or, for a room, the event) whose chunks the collection +//! holds. It lives in the collection's own metadata, so it is lost with the collection //! and the two cannot disagree about what is held. It is advanced only after -//! the repo's deletes and upserts are acknowledged: it may lag and never leads. -//! It is dropped before the repo's first delete, so a run that fails between -//! the delete and the upserts leaves no cursor, and the repo is ingested whole. -//! The cursors object is written whole by this job alone, once per repo. +//! the source's deletes and upserts are acknowledged: it may lag and never leads. +//! A repo's is dropped before its first delete, so a run that fails between +//! the delete and the upserts leaves no cursor, and the repo is ingested whole; a room's +//! chat chunks are written before anything they replace is deleted. The cursors +//! object is written whole by this job alone. A key is a repo name, or a chat key +//! (`chat::is_chat_key`), never both. +pub mod chat; +pub mod chat_chunk; +pub mod chat_purge; pub mod chunk; pub mod forge; pub mod git; +pub mod room_scope; +#[cfg(test)] +mod chat_tests; #[cfg(test)] mod tests; @@ -190,6 +201,14 @@ impl ChatFilter { } } +/// A chat chunk as the index holds it, with its point id: what the ingest job +/// reads back to rewrite a chunk. +#[derive(Clone, Debug, PartialEq)] +pub struct StoredChat { + pub id: String, + pub payload: ChatPayload, +} + /// One chunk a vector search returned and how near it scored. #[derive(Clone, Debug, PartialEq)] pub struct SourceRow { @@ -227,6 +246,16 @@ pub trait SourceIndex { /// Up to `limit` chat chunks `filter` keeps, by span start in `order`. /// Reads only `corpus=chat` points; the daemon's. fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result>; + /// The room's chunk with the greatest `ts_end_ms`, if it has any. The ingest job's alone. + fn chat_tail(&mut self, room_id: &str) -> Result>; + /// The room's chunks holding any of `event_ids`. The ingest job's alone. + fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result>; + /// The room's chunks whose span reaches into [`from_ms`, `to_ms`): `ts_end_ms >= from_ms` + /// and `ts_start_ms < to_ms`. The ingest job's alone. + fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result>; + /// Deletes a room's chat chunks: those with the point ids `ids`, or all of them when `None`. + /// Acknowledged before it returns. Never touches a forge point. The ingest job's alone. + fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()>; } /// A repo the forge lists as indexable. @@ -395,7 +424,7 @@ fn retire( failed: &mut Vec, ) { let listed = |name: &str| catalog.iter().any(|repo| repo.name == name); - let mut gone: BTreeSet = cursors.keys().filter(|name| !listed(name)).cloned().collect(); + let mut gone: BTreeSet = cursors.keys().filter(|name| !chat::is_chat_key(name) && !listed(name)).cloned().collect(); if let Ok(entries) = std::fs::read_dir(mirrors) { for entry in entries.flatten() { if let Some(name) = entry.file_name().to_string_lossy().strip_suffix(".git") diff --git a/crates/huginn-daemon/src/sources/room_scope.rs b/crates/huginn-daemon/src/sources/room_scope.rs new file mode 100644 index 0000000..8d7cf2d --- /dev/null +++ b/crates/huginn-daemon/src/sources/room_scope.rs @@ -0,0 +1,184 @@ +//! Which joined rooms and which senders are in the chat corpus. The only code +//! that decides it, and the only file outside tests that names the bridged +//! platform whose content is refused. +//! +//! Scope (ruling `gamecult-chat:ruling:chat-index-minds-rooms`): the rooms the +//! reader user has joined, minus this exclusion. A room's join rule is read by +//! nobody (`RoomFacts` has no field for it): a joined invite-only room qualifies +//! like any other. The reader roster is the first lock; this file is the second. + +use std::collections::BTreeSet; + +use super::chat::RoomEvent; + +/// The estate rooms by canonical alias, ASCII case-insensitive. Code owns the +/// aliases and the rule; the unit supplies the room ids (`Estate`), and either alone excludes. +pub const ESTATE_ALIASES: [&str; 3] = ["#users:gamecult.org", "#members:gamecult.org", "#patrons:gamecult.org"]; + +/// What the server says about a room, as far as admission needs it. The Matrix +/// client fills it from the room's state and decides nothing. It has no `Default`: a +/// room whose state could not be read is a port error, never a plain room. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoomFacts { + pub room_id: String, + pub alias: Option, + pub alt_aliases: Vec, + pub name: Option, + /// An `m.room.encryption` state event exists. + pub encrypted: bool, + /// Any `m.room.member` in the state has `is_direct` true. + pub direct: bool, + /// The state type (`m.bridge` or `uk.half-shot.bridge`) of a bridge event, when the room has one. + pub bridge: Option, +} + +/// The estate room ids the caller knows (they exist only on the live server): exactly one per +/// estate alias, so a job that was handed none, or too few, cannot be built. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Estate { + ids: BTreeSet, +} + +impl Estate { + pub fn new(ids: BTreeSet) -> anyhow::Result { + anyhow::ensure!( + ids.len() == ESTATE_ALIASES.len() && ids.iter().all(|id| id.starts_with('!')), + "the estate is exactly {} room ids ({}), got {}", + ESTATE_ALIASES.len(), + ESTATE_ALIASES.join(", "), + ids.len() + ); + Ok(Self { ids }) + } + + /// Whether `room_id` is a named estate room. Decided before the room's state is read. + pub fn holds_id(&self, room_id: &str) -> bool { + self.ids.contains(room_id) + } +} + +/// Why a joined room, or an event in it, is refused. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Refusal { + Estate, + Direct, + Encrypted, + Bridged(String), + DiscordSender, + DiscordContent, +} + +impl std::fmt::Display for Refusal { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Estate => formatter.write_str("estate room"), + Self::Direct => formatter.write_str("direct room"), + Self::Encrypted => formatter.write_str("encrypted room"), + Self::Bridged(state_type) => write!(formatter, "bridged room ({state_type})"), + Self::DiscordSender => formatter.write_str("discord sender"), + Self::DiscordContent => formatter.write_str("discord content"), + } + } +} + +/// Whether a joined room may be read at all. +pub fn admit_room(facts: &RoomFacts, estate: &Estate) -> Result<(), Refusal> { + let estate_alias = |alias: &str| ESTATE_ALIASES.iter().any(|estate| estate.eq_ignore_ascii_case(alias)); + if estate.holds_id(&facts.room_id) || facts.alias.iter().chain(&facts.alt_aliases).any(|alias| estate_alias(alias)) { + return Err(Refusal::Estate); + } + if facts.direct { + return Err(Refusal::Direct); + } + if facts.encrypted { + return Err(Refusal::Encrypted); + } + if let Some(state_type) = &facts.bridge { + return Err(Refusal::Bridged(state_type.clone())); + } + Ok(()) +} + +/// Whether an event refuses its whole room: a sender whose localpart, lowercased, starts with the +/// platform's name after any leading underscores, has it after an underscore (`mautrix_discord`, +/// `_bridge_discordbot`), or has it followed by `bot` or an underscore; or content with a top-level key +/// naming it. Deterministic identifier rules, not language reading; `discourse` is not it. +pub fn refuse_event(event: &RoomEvent) -> Option { + let localpart = event.sender.trim_start_matches('@').split(':').next().unwrap_or("").to_ascii_lowercase(); + let name = ["disc", "ord"].concat(); + if localpart.trim_start_matches('_').starts_with(&name) || localpart.contains(&format!("_{name}")) || localpart.contains(&format!("{name}bot")) || localpart.contains(&format!("{name}_")) { + return Some(Refusal::DiscordSender); + } + event.content_keys.iter().any(|key| key.to_ascii_lowercase().contains("discord")).then_some(Refusal::DiscordContent) +} + +#[cfg(test)] +mod tests { + use super::super::chat::EventKind; + use super::*; + + fn facts(id: &str) -> RoomFacts { + RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None } + } + + fn estate(ids: &[&str]) -> Estate { + Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap() + } + + fn event(sender: &str, keys: &[&str]) -> RoomEvent { + RoomEvent { event_id: "$e".into(), sender: sender.into(), ts_ms: 1, content_keys: keys.iter().map(|key| (*key).into()).collect(), kind: EventKind::Other } + } + + #[test] + fn a_direct_encrypted_or_bridged_room_is_refused_and_a_plain_room_is_admitted() { + let estate = estate(&[]); + assert_eq!(admit_room(&RoomFacts { direct: true, ..facts("!a:s") }, &estate), Err(Refusal::Direct)); + assert_eq!(admit_room(&RoomFacts { encrypted: true, ..facts("!a:s") }, &estate), Err(Refusal::Encrypted)); + for state_type in ["m.bridge", "uk.half-shot.bridge"] { + assert_eq!(admit_room(&RoomFacts { bridge: Some(state_type.into()), ..facts("!a:s") }, &estate), Err(Refusal::Bridged(state_type.into()))); + } + assert_eq!(admit_room(&RoomFacts { alias: Some("#swarm:gamecult.org".into()), name: Some("Swarm".into()), ..facts("!a:s") }, &estate), Ok(())); + } + + #[test] + fn the_estate_is_refused_by_id_and_by_alias_in_any_case_and_a_lookalike_is_not() { + let named = estate(&["!ids:s"]); + assert!(named.holds_id("!ids:s") && !named.holds_id("!other:s")); + assert_eq!(admit_room(&facts("!ids:s"), &named), Err(Refusal::Estate)); + for alias in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:Gamecult.Org".to_owned()]) { + assert_eq!(admit_room(&RoomFacts { alias: Some(alias.clone()), ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "{alias}"); + assert_eq!(admit_room(&RoomFacts { alt_aliases: vec!["#a:s".into(), alias.clone()], ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "alt {alias}"); + } + for lookalike in ["#users-guide:gamecult.org", "#patrons-club:gamecult.org", "#users:gamecult.org.evil", "#users:other.org"] { + assert_eq!(admit_room(&RoomFacts { alias: Some(lookalike.into()), ..facts("!x:s") }, &estate(&[])), Ok(()), "{lookalike}"); + } + } + + #[test] + fn a_bridged_platform_sender_or_content_key_refuses_and_a_lookalike_does_not() { + for sender in ["@discord_123:s", "@_discord_9:s", "@__DiscordBot:s", "@discordbot:s", "@mautrix_discord:s", "@_bridge_discordbot:s", "@a_discord:s"] { + assert_eq!(refuse_event(&event(sender, &[])), Some(Refusal::DiscordSender), "{sender}"); + } + for key in ["fi.mau.discord", "uk.half-shot.discord", "X.DISCORD.y"] { + assert_eq!(refuse_event(&event("@a:s", &["body", key])), Some(Refusal::DiscordContent), "{key}"); + } + for sender in ["@discourse:s", "@disco:s", "@eureka:s", "@x:discord.com"] { + assert_eq!(refuse_event(&event(sender, &["discourse", "body"])), None, "{sender}"); + } + } + + #[test] + fn the_estate_needs_exactly_its_room_ids() { + for ids in [vec![], vec!["!a:s"], vec!["!a:s", "!b:s", "!c:s", "!d:s"], vec!["!a:s", "!b:s", "notaroom"]] { + assert!(Estate::new(ids.iter().map(|id| (*id).to_owned()).collect()).is_err(), "{ids:?}"); + } + assert!(Estate::new(["!a:s", "!b:s", "!c:s"].map(str::to_owned).into()).is_ok()); + } + + #[test] + fn a_room_facts_carries_no_join_rule() { + let source = include_str!("room_scope.rs"); + let code = &source[..source.find("#[cfg(test)]").unwrap()]; + assert!(!code.contains(&["join", "_rule"].concat())); + } +} diff --git a/crates/huginn-daemon/src/sources/tests.rs b/crates/huginn-daemon/src/sources/tests.rs index fbb364e..a7203a4 100644 --- a/crates/huginn-daemon/src/sources/tests.rs +++ b/crates/huginn-daemon/src/sources/tests.rs @@ -1155,3 +1155,24 @@ fn a_window_asks_the_index_only_and_needs_no_embedder() { let signature = &signature[..signature.find('{').unwrap()]; assert!(!signature.contains("Embedder"), "{signature}"); } + +/// A chat cursor and a purge tombstone share the forge's cursors object; the +/// forge retire never reads either as a repo name, and still retires a repo. +#[test] +fn chat_cursor_and_purge_tombstone_keys_survive_the_forge_retire() { + let (keep, gone) = (Remote::new(), Remote::new()); + keep.commit(&[("a.rs", b"a\n")], &[]); + gone.commit(&[("b.rs", b"b\n")], &[]); + let mut rig = Rig::new(); + rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("Gone", &gone)])); + let mut cursors = rig.index.cursors(); + cursors.insert("chat:!r:s".into(), "1:$e".into()); + cursors.insert("chat-purge:!r:s".into(), "1..2".into()); + rig.index.set_cursors(&cursors).unwrap(); + rig.index.upsert_chat(&[chat_point("!r:s", None, 0, 4, &[("$e", 5)])]).unwrap(); + rig.converged(&mut FakeForge::of(&[("Keep", &keep)])); + let after = rig.index.cursors(); + assert_eq!(after.keys().map(String::as_str).collect::>(), ["Keep", "chat-purge:!r:s", "chat:!r:s"], "Gone retired; both chat keys kept"); + assert!(rig.index.state.lock().unwrap().deletes.iter().all(|(repo, _)| !repo.starts_with("chat"))); + assert_eq!(rig.index.state.lock().unwrap().chat.len(), 1); +} diff --git a/crates/huginn-mind/src/lib.rs b/crates/huginn-mind/src/lib.rs index 0e41aa9..56dbf3c 100644 --- a/crates/huginn-mind/src/lib.rs +++ b/crates/huginn-mind/src/lib.rs @@ -41,6 +41,7 @@ mod rows; pub mod oracle; pub mod query; pub mod receipt; +pub mod redact; pub mod refusal; pub mod store; pub mod wire; diff --git a/crates/huginn-mind/src/redact.rs b/crates/huginn-mind/src/redact.rs new file mode 100644 index 0000000..23ccf57 --- /dev/null +++ b/crates/huginn-mind/src/redact.rs @@ -0,0 +1,514 @@ +//! Credential redaction: the only code that knows the shape of a secret in text. +//! +//! `redact_url_userinfo` redacts URL credentials (the Self run's journal and reply, +//! the Persona's structured output). `scrub` is chat ingest's door: every message body +//! passes it before it is chunked, embedded or stored, and it also runs the URL rule. +//! Shapes only; nothing here reads meaning, and a hit is a kind, never a value. + +/// The one redaction of URL credentials. Call it at each place a string the CLI produced enters the journal, her mind or chat: today +/// the Self result (`result_text`: journal line and Reply) and the structured output of a Persona or Sleep result +/// (`persona::structured_output`, string by string). It does not cover claude's stderr (inherited) or the CLI's transcript files. +/// +/// A forge token rides in a remote URL, so every `scheme://` and every protocol-relative `//` opens an authority, read as git, curl and +/// RFC 3986 read it: it ends at the first `/`, `?`, `#`, backslash, quote, backtick, parenthesis, bracket, angle bracket or ASCII whitespace +/// (`is_authority_end`). Its userinfo is everything before the authority's LAST `@` (a raw `@` in a password still redacts), and non-ASCII +/// characters, NBSP and U+2028 included, stay inside it. The userinfo becomes ``: `scheme://@host`. With a colon it is +/// always redacted; without one it is a bare token on http, https, ftp, ftps, ws, wss, any `+http`/`+https` scheme (`git+https`), protocol-relative `//` and an unparseable scheme, and a plain +/// username on any other scheme (`ssh://git@host`), which stays. A `@` after the authority (`github.com/@user`, `pkg@1.2.3`) is path, not +/// userinfo, and is left alone. Scheme-less credentials are covered only in the shape `word:secret@host.tld/` at the start of an +/// ASCII-whitespace-delimited word. Not covered: a password holding `/ ? # ` a quote or ASCII whitespace (no parser reads it as userinfo), +/// scp-style `user:secret@host:path`, tokens in query strings, claude's stderr, the kept transcripts. Syntax only; no reading of meaning. +pub fn redact_url_userinfo(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(at) = rest.find("//") { + let (before, after) = rest.split_at(at); + let body = &after[2..]; + out.push_str(before); + out.push_str("//"); + // The scheme is the ASCII run before `:`. Without a colon, `//` opens an authority only at a word start (`a//b@c` is not one). + let scheme = match before.strip_suffix(':') { + // Only ASCII bytes match, so the byte after the last non-match is a character boundary. + Some(head) => head[head.bytes().rposition(|c| !(c.is_ascii_alphanumeric() || matches!(c, b'+' | b'-' | b'.'))).map_or(0, |i| i + 1)..].to_ascii_lowercase(), + None if before.chars().next_back().is_none_or(|c| is_break(c) || matches!(c, '"' | '\'' | '`' | '(' | '[' | '{' | '<' | '=' | ',')) => String::new(), + None => { + rest = body; + continue; + } + }; + let end = body.find(is_authority_end).unwrap_or(body.len()); + let authority = &body[..end]; + match authority.rfind('@') { + Some(i) if i > 0 && (authority[..i].contains(':') || carries_bare_token(&scheme)) => { + out.push_str(""); + out.push_str(&authority[i..]); + } + _ => out.push_str(authority), + } + rest = &body[end..]; + } + out.push_str(rest); + redact_schemeless(&out) +} + +/// Schemes whose lone userinfo is a token, not a username: web and ftp schemes, `ws`/`wss`, and `x+http`/`x+https` wrappers; `scheme` is lowercase. +fn carries_bare_token(scheme: &str) -> bool { + matches!(scheme, "" | "http" | "https" | "ftp" | "ftps" | "ws" | "wss") || scheme.ends_with("+http") || scheme.ends_with("+https") +} + +/// ASCII whitespace; NBSP, U+2028 and other Unicode whitespace are not, so a secret holding them is redacted whole. +fn is_break(c: char) -> bool { + matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c') +} + +/// The end of a URL's authority. +fn is_authority_end(c: char) -> bool { + is_break(c) || matches!(c, '/' | '?' | '#' | '\\' | '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | '<' | '>') +} + +/// `word:secret@host.tld/` at the start of a whitespace-delimited word that holds no `//` becomes `@host.tld/`. +fn redact_schemeless(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut word = String::new(); + let flush = |word: &mut String, out: &mut String| { + out.push_str(&redact_word(word)); + word.clear(); + }; + for c in text.chars() { + if is_break(c) { + flush(&mut word, &mut out); + out.push(c); + } else { + word.push(c); + } + } + flush(&mut word, &mut out); + out +} + +fn redact_word(word: &str) -> String { + let shaped = (|| { + if word.contains("//") { + return None; + } + let at = word.rfind('@')?; + let colon = word.find(':')?; + let (user, host) = (&word[..colon], &word[at + 1..]); + let host_name = &host[..host.find('/')?]; + let user_ok = !user.is_empty() && user.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')); + let host_ok = host_name.contains('.') && host_name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-')); + (colon + 1 < at && user_ok && host_ok).then_some(at) + })(); + match shaped { + Some(at) => format!("{}", &word[at..]), + None => word.to_string(), + } +} + +/// The version of the scrub's rules. Part of the chat version: a bump makes every +/// room re-ingest from the room's history and scrub it again. +pub const SCRUB_VERSION: u32 = 2; + +/// What replaces a secret. +pub const REDACTED: &str = ""; + +/// The shape of secret a scrub found. Never its value. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum ScrubKind { + UrlUserinfo, + Bearer, + AuthorizationHeader, + KeyShape, + Jwt, + PemBlock, + AccessToken, +} + +impl std::fmt::Display for ScrubKind { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::UrlUserinfo => "url-userinfo", + Self::Bearer => "bearer", + Self::AuthorizationHeader => "authorization-header", + Self::KeyShape => "key-shape", + Self::Jwt => "jwt", + Self::PemBlock => "pem-block", + Self::AccessToken => "access-token", + }) + } +} + +/// A body with its secrets replaced by ``, and the kinds found, one per secret. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Scrubbed { + pub text: String, + pub hits: Vec, +} + +/// Zero-width characters a secret could be split with. +fn zero_width(c: char) -> bool { + matches!(c, '\u{200b}' | '\u{200c}' | '\u{200d}' | '\u{2060}' | '\u{feff}' | '\u{00ad}' | '\u{180e}') +} + +/// Replaces credential shapes in `text`: a PEM or PGP private key block, an `authorization:` header +/// line, `Bearer`/`Basic`/`token` followed by a long token (blank or colon between), a vendor key +/// shape (Matrix tokens included), an `access_token` query or JSON value, a JWT, and URL userinfo. +/// Each hit becomes ``. Token shapes only; ordinary prose, SHAs, UUIDs and ids pass byte for byte. +/// Zero-width characters are removed before matching; when anything matched, the returned text is the +/// stripped one, so a token split by them leaves no tail, and when nothing matched the text is returned as it came (emoji joiners stay). +pub fn scrub(text: &str) -> Scrubbed { + let mut hits = Vec::new(); + let mut stripped: String = text.chars().filter(|c| !zero_width(*c)).collect(); + let passes: [(ScrubKind, fn(&str, usize) -> Option); 6] = [ + (ScrubKind::PemBlock, pem_block), + (ScrubKind::AuthorizationHeader, authorization_line), + (ScrubKind::AccessToken, access_token), + (ScrubKind::Bearer, bearer_token), + (ScrubKind::KeyShape, key_shape), + (ScrubKind::Jwt, jwt), + ]; + for (kind, find) in passes { + stripped = replace_matches(&stripped, kind, find, &mut hits); + } + let urls = redact_url_userinfo(&stripped); + let (before, after) = (stripped.matches(REDACTED).count(), urls.matches(REDACTED).count()); + hits.extend(std::iter::repeat_n(ScrubKind::UrlUserinfo, after.saturating_sub(before))); + if hits.is_empty() { Scrubbed { text: text.to_owned(), hits } } else { Scrubbed { text: urls, hits } } +} + +fn replace_matches(text: &str, kind: ScrubKind, find: fn(&str, usize) -> Option, hits: &mut Vec) -> String { + let mut out = String::with_capacity(text.len()); + let mut at = 0; + while at < text.len() { + match find(text, at) { + Some(end) if end > at => { + out.push_str(REDACTED); + hits.push(kind); + at = end; + } + _ => { + let next = at + text[at..].chars().next().map_or(1, char::len_utf8); + out.push_str(&text[at..next]); + at = next; + } + } + } + out +} + +/// Whether `at` starts a word: the byte before it is not an ASCII letter or digit. +fn word_start(text: &str, at: usize) -> bool { + at == 0 || !text.as_bytes()[at - 1].is_ascii_alphanumeric() +} + +/// The end of the run of bytes at `from` that `member` accepts. +fn run_end(text: &str, from: usize, member: impl Fn(u8) -> bool) -> usize { + from + text.as_bytes()[from..].iter().take_while(|byte| member(**byte)).count() +} + +fn starts_with_ignore_case(text: &str, at: usize, word: &str) -> bool { + text.as_bytes().get(at..at + word.len()).is_some_and(|held| held.eq_ignore_ascii_case(word.as_bytes())) +} + +fn token_char(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'+' | b'/' | b'=' | b'-') +} + +fn key_char(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-') +} + +/// `-----BEGIN ... PRIVATE KEY-----` through its `-----END ... PRIVATE KEY-----`, or the text's end. +fn pem_block(text: &str, at: usize) -> Option { + let rest = &text[at..]; + let label = rest.strip_prefix("-----BEGIN ")?; + let words = run_end(label, 0, |byte| byte.is_ascii_uppercase() || byte == b' '); + (label[..words].ends_with("PRIVATE KEY") || label[..words].ends_with("PRIVATE KEY BLOCK")).then_some(())?; + label[words..].starts_with("-----").then_some(())?; + let body_from = at + "-----BEGIN ".len() + words + 5; + let end = text[body_from..].find("-----END ").and_then(|end| { + let name = body_from + end + "-----END ".len(); + let words = run_end(text, name, |byte| byte.is_ascii_uppercase() || byte == b' '); + text[words..].starts_with("-----").then_some(words + 5) + }); + Some(end.unwrap_or(text.len())) +} + +/// `authorization:` in any case at a word start, through the end of the line. +fn authorization_line(text: &str, at: usize) -> Option { + (word_start(text, at) && starts_with_ignore_case(text, at, "authorization:")).then_some(())?; + let from = at + "authorization:".len(); + let end = run_end(text, from, |byte| byte != b'\n' && byte != b'\r'); + (!text[from..end].trim().is_empty()).then_some(end) +} + +/// `Bearer`, `Basic` or `token`, blanks, then 16 or more token characters. +fn bearer_token(text: &str, at: usize) -> Option { + word_start(text, at).then_some(())?; + let word = ["bearer", "basic", "token"].into_iter().find(|word| starts_with_ignore_case(text, at, word))?; + let blanks = at + word.len(); + let colon = usize::from(text.as_bytes().get(blanks) == Some(&b':')); + let value = run_end(text, blanks + colon, |byte| byte == b' ' || byte == b'\t'); + (value > blanks).then_some(())?; + let end = run_end(text, value, token_char); + (end - value >= 16).then_some(end) +} + +/// A vendor key: the prefix, then a long enough run of the key alphabet. +fn key_shape(text: &str, at: usize) -> Option { + let rest = &text[at..]; + let by_prefix = [("sk-", 16), ("sk_live_", 16), ("pk_live_", 16), ("github_pat_", 16), ("glpat-", 16), ("syt_", 16), ("mct_", 16)]; + let tokens = ["ghp_", "gho_", "ghu_", "ghs_", "ghr_"].into_iter().map(|prefix| (prefix, 16)); + for (prefix, least) in by_prefix.into_iter().chain(tokens) { + // Only the generic prefix needs a word start (`risk-averse`, `task-force`); the distinctive ones are found glued to anything. + if rest.starts_with(prefix) && (prefix != "sk-" || word_start(text, at)) { + let end = run_end(text, at + prefix.len(), key_char); + return (end - at - prefix.len() >= least).then_some(end); + } + } + for prefix in ["xoxa-", "xoxb-", "xoxp-", "xoxr-", "xoxs-"] { + if rest.starts_with(prefix) { + let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_alphanumeric() || byte == b'-'); + return (end - at - prefix.len() >= 16).then_some(end); + } + } + for prefix in ["AKIA", "ASIA"] { + if rest.starts_with(prefix) { + let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_uppercase() || byte.is_ascii_digit()); + return (end - at - prefix.len() >= 16).then_some(end); + } + } + if rest.starts_with("AIza") { + let end = run_end(text, at + 4, key_char); + return (end - at - 4 >= 35).then_some(end); + } + None +} + +/// `access_token` as a query parameter or JSON member, through its value (8 or more characters). +fn access_token(text: &str, at: usize) -> Option { + starts_with_ignore_case(text, at, "access_token").then_some(())?; + let mut at = at + "access_token".len(); + let skip = |at: usize, set: &[u8]| run_end(text, at, |byte| set.contains(&byte)); + at = skip(at, b"\"'"); + at = skip(at, b" \t"); + matches!(text.as_bytes().get(at), Some(b'=' | b':')).then_some(())?; + at = skip(at + 1, b" \t"); + at = skip(at, b"\"'"); + let end = run_end(text, at, |byte| !byte.is_ascii_whitespace() && !matches!(byte, b'&' | b'"' | b'\'' | b',' | b'}' | b';' | b'#')); + (end - at >= 8).then_some(end) +} + +/// Three base64url segments, the first starting `eyJ`. +fn jwt(text: &str, at: usize) -> Option { + text[at..].starts_with("eyJ").then_some(())?; + let segment = |from: usize| run_end(text, from, key_char); + let first = segment(at); + let mut end = first; + for _ in 0..2 { + (text.as_bytes().get(end) == Some(&b'.')).then_some(())?; + let next = segment(end + 1); + (next > end + 1).then_some(())?; + end = next; + } + Some(end) +} + +#[cfg(test)] +mod tests { + use super::*; + + const CANARY: &str = "CANARY-NOT-A-REAL-TOKEN-0000"; + + #[test] + fn url_userinfo_is_redacted_where_it_is_a_secret_and_kept_where_it_is_a_username() { + let said = format!( + "a https://user:{CANARY}@host.example/r.git b https://{CANARY}@host.example/r.git c ssh://u:{CANARY}@h/x d http://{CANARY}@h e HTTPS://x:{CANARY}@h \ + f https://u:p@ss{CANARY}@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://u:{CANARY}@h) k ://@x" + ); + let want = "a https://@host.example/r.git b https://@host.example/r.git c ssh://@h/x d http://@h e HTTPS://@h \ + f https://@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://@h) k ://@x"; + assert_eq!(redact_url_userinfo(&said), want); + assert!(!redact_url_userinfo(&said).contains(CANARY)); + } + + #[test] + fn a_secret_is_redacted_whole_by_the_authority_rule_and_commonplace_urls_are_left_alone() { + let r = redact_url_userinfo; + // Raw @, percent-encoding and non-ASCII (NBSP, U+2028) stay inside userinfo. + for secret in ["pa@ssT", "pa%40ssT", "pa\u{a0}ssT", "pa\u{2028}ssT", "pa:ssT", "\u{e9}\u{65e5}ssT"] { + let out = r(&format!("x https://u:{secret}@h/x")); + assert_eq!(out, "x https://@h/x", "{secret:?}"); + } + // The authority ends at / ? # \\ quotes, brackets and ASCII whitespace: no parser reads those as userinfo, so what follows is path. + assert_eq!(r("https://u:pa/ssT@h/x"), "https://u:pa/ssT@h/x"); + assert_eq!(r("https://u:pa ssT@h/x"), "https://u:pa ssT@h/x"); + assert_eq!(r("https://u:pa?ssT@h"), "https://u:pa?ssT@h"); + // Ordinary URLs with @ in the path are byte-identical. + for same in [ + "https://github.com/@user/repo", + "https://medium.com/@name/post-1", + "https://registry.npmjs.org/@scope/pkg", + "https://h/pkg@1.2.3", + "a//b@c", + "see //cdn.h/@x/y and https://h/?mail=a@b.c#f@g", + "https://h:8080/p git@github.com:org/r.git ssh://git@host/r.git ssh://git@h:22/r git://git@h/x", + ] { + assert_eq!(r(same), same); + } + // Compact JSON: only the credentialed URL changes. + let json = format!(r#"{{"a":"https://u:{CANARY}@h/x","b":"https://h2/@v","c":"https://h3/p@1"}}"#); + assert_eq!(r(&json), r#"{"a":"https://@h/x","b":"https://h2/@v","c":"https://h3/p@1"}"#); + assert_eq!(r("a/https://u:T@h/x,https://v:T@h2"), "a/https://@h/x,https://@h2"); + assert_eq!(r("HTTPS://TOKEN@h"), "HTTPS://@h"); + assert_eq!(r("[l](https://T@h/x) `https://u:T@h`"), "[l](https://@h/x) @h> `https://@h`"); + assert_eq!(r("a //u:T@host/x b //T@host c \"//T@h\" (//T@h)"), "a //@host/x b //@host c \"//@h\" (//@h)"); + // Token-bearing schemes beyond http: +http/+https wrappers and ws/wss redact a bare token; git+ssh and ssh keep their username. + assert_eq!(r("git+https://T@github.com/a/b GIT+HTTPS://T@h wss://T@h/ ws://T@h/"), "git+https://@github.com/a/b GIT+HTTPS://@h wss://@h/ ws://@h/"); + for same in ["git+ssh://git@host/x", "ssh://git@host/x"] { + assert_eq!(r(same), same); + } + // Scheme-less: only word:secret@host.tld/ . Each guard of the shape has its own case. + assert_eq!(r("user:T@host.tld/path"), "@host.tld/path"); + for same in ["a:b@c", "a:b@host/x", "a:b@ho_st.tld/x", "u$er:T@host.tld/x", "a:@host.tld/x", "mail me@x.org/ and :T@host.tld/x"] { + assert_eq!(r(same), same); + } + } + + #[test] + fn multibyte_text_never_panics_the_redaction() { + let r = redact_url_userinfo; + for case in ["see \u{201c}https://h/x", "\u{2192}https://h/x", "docs\u{ff1a}https://h/x", "\u{e9}://x", "\u{65e5}\u{672c}://u:T@h", "https://x/\u{e9}://", "\u{e9}//u:T@h", "\u{e9}:u:T@h.tld/"] { + let _ = r(case); + } + assert_eq!(r("\u{65e5}\u{672c}://u:T@h"), "\u{65e5}\u{672c}://@h"); + assert_eq!(r("\u{2192}https://u:T@h/x"), "\u{2192}https://@h/x"); + let seed = "\u{201c}\u{2192}\u{ff1a}\u{e9}\u{65e5}\u{a0}https://u:T@h/x//y:z@a.b/ ://@ //"; + let chars: Vec = seed.chars().collect(); + for start in 0..=chars.len() { + for end in start..=chars.len() { + let _ = r(&chars[start..end].iter().collect::()); + } + } + } + + + /// Every shape is built here from fake parts; none is a real credential. + fn secrets() -> Vec<(ScrubKind, String, String)> { + let pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfakefakefake\n-----END RSA PRIVATE KEY-----".to_owned(); + vec![ + (ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2".into()), + (ScrubKind::Bearer, format!("Bearer {}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()), + (ScrubKind::Bearer, format!("Basic {}", "dXNlcjpwYXNz".repeat(2)), "dXNlcjpwYXNzdXNl".into()), + (ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3".into()), + (ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5".into()), + (ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9".into()), + (ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2".into()), + (ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7".into()), + (ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()), + (ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()), + (ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_".into()), + (ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ".into()), + (ScrubKind::PemBlock, pem, "MIIEowIBAAKCAQEAfake".into()), + (ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake".into()), + (ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR".into()), + (ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5".into()), + (ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()), + (ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()), + (ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()), + (ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()), + (ScrubKind::KeyShape, "ASIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()), + (ScrubKind::KeyShape, "gho_".to_owned() + &"p1Q2".repeat(9), "p1Q2p1Q2p1Q2".into()), + (ScrubKind::KeyShape, "pk_live_".to_owned() + &"e3R4".repeat(6), "e3R4e3R4e3R4".into()), + (ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz".into()), + (ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654".into()), + (ScrubKind::Bearer, format!("bearer:{}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()), + (ScrubKind::Bearer, format!("token: {}", "m1N2o3".repeat(4)), "m1N2o3m1N2".into()), + (ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake".into()), + (ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2".into()), + (ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345".into()), + ] + } + + #[test] + fn every_planted_credential_is_replaced_mid_line_and_at_line_end_with_its_kind() { + for (kind, secret, distinct) in secrets() { + for body in [format!("before {secret} after"), format!("before {secret}"), format!("{secret}\nnext line")] { + let out = scrub(&body); + assert!(!out.text.contains(&distinct), "{kind}: {}", out.text); + assert!(out.text.contains(REDACTED), "{kind}: {}", out.text); + assert!(out.hits.contains(&kind), "{kind} not in {:?} for {body:?}", out.hits); + } + } + } + + #[test] + fn a_pem_block_without_its_end_runs_to_the_end_of_the_body_and_one_with_it_stops_there() { + let open = scrub("see -----BEGIN EC PRIVATE KEY-----\nAAAA\nBBBB").text; + assert_eq!(open, "see "); + let closed = scrub("-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY----- and then prose").text; + assert_eq!(closed, " and then prose"); + } + + #[test] + fn an_authorization_header_goes_to_the_end_of_its_line_only() { + assert_eq!(scrub("hi\nAUTHORIZATION: Token abc\nbye").text, "hi\n\nbye"); + assert_eq!(scrub("Authorization: x").hits, [ScrubKind::AuthorizationHeader]); + } + + #[test] + fn ordinary_text_passes_byte_for_byte_and_reports_no_hit() { + for same in [ + "risk-averse", "task-force", "sk-", "sk-short", "3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c", + "123e4567-e89b-12d3-a456-426614174000", "!abc:s", "$Ab3dEf-9_zQ", "user@example.com", "ssh://git@host/r.git", + "https://github.com/@user", "pkg@1.2.3", "the bearer of bad news", "Authorization is a header", + "padding like dGVzdA== in prose", "```\nlet x = 1;\n```", "the BEGIN PRIVATE KEY header", "a token of thanks", + "AKIAshort", "ghp_short", "eyJ is a prefix", "naïve café — 日本語 with Bearer in it", + ] { + let out = scrub(same); + assert_eq!(out.text, same); + assert!(out.hits.is_empty(), "{same}: {:?}", out.hits); + } + } + + #[test] + fn multibyte_text_around_a_secret_never_panics_the_scrub() { + let body = format!("日本 \u{a0}é Bearer {} → ünï", "A1b2".repeat(6)); + assert_eq!(scrub(&body).text, "日本 \u{a0}é → ünï"); + let seed = "é日 sk-aaaaaaaaaaaaaaaaaaaa -----BEGIN PRIVATE KEY----- authorization: é eyJa.b.c https://u:p@h/x"; + let chars: Vec = seed.chars().collect(); + for start in 0..chars.len() { + for end in start..=chars.len() { + let _ = scrub(&chars[start..end].iter().collect::()); + } + } + } + + #[test] + fn a_distinctive_key_is_found_glued_to_other_text_but_the_generic_prefix_still_needs_a_word_start() { + let ghp = format!("xxx{}", "ghp_".to_owned() + &"k2L9".repeat(9)); + assert_eq!(scrub(&ghp).text, "xxx"); + assert_eq!(scrub("idAKIA0123456789ABCDEFend").hits, [ScrubKind::KeyShape]); + assert_eq!(scrub(&format!("zzz{}", "xoxb-123456789012-aBcDeFgHiJkLmNoP")).hits, [ScrubKind::KeyShape]); + assert_eq!(scrub(&format!("q{}", "AIza".to_owned() + &"Sy1_".repeat(9))).hits, [ScrubKind::KeyShape]); + assert!(scrub("risk-averse-thinking-process task-force-alpha-bravo").hits.is_empty()); + } + + #[test] + fn zero_width_characters_inside_a_token_leave_no_tail_and_emoji_joiners_survive_when_nothing_matched() { + let split = format!("sk-{}\u{200b}{}", "q7W3".repeat(4), "q7W3".repeat(4)); + let out = scrub(&format!("key {split} end")); + assert_eq!(out.text, "key end"); + let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJz\u{feff}dWIiOiJmYWtlIn0.c2lnbmF0dXJl\u{200d}ZmFrZQ"; + assert_eq!(scrub(jwt).text, ""); + let family = "family \u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467} here"; + assert_eq!(scrub(family).text, family); + } + + #[test] + fn matrix_tokens_and_query_values_are_found_and_prose_about_them_is_not() { + for same in ["the access_token is a thing", "access_token: short", "syt_ is the prefix", "mct_ tokens", "bearer:short"] { + assert_eq!(scrub(same).text, same, "{same}"); + } + } +} diff --git a/crates/mind-body/src/persona.rs b/crates/mind-body/src/persona.rs index 03935b6..28e2e1d 100644 --- a/crates/mind-body/src/persona.rs +++ b/crates/mind-body/src/persona.rs @@ -1964,7 +1964,7 @@ fn refused_note(why: Failure) -> String { /// Every string of `value` through the one URL-credential redaction, so JSON structure is never inside a candidate run. fn redact_strings(value: &mut Value) { match value { - Value::String(text) => *text = crate::self_run::redact_url_userinfo(text), + Value::String(text) => *text = huginn_mind::redact::redact_url_userinfo(text), Value::Array(items) => items.iter_mut().for_each(redact_strings), Value::Object(map) => map.values_mut().for_each(redact_strings), _ => {} diff --git a/crates/mind-body/src/self_run.rs b/crates/mind-body/src/self_run.rs index 754b93c..fc2016c 100644 --- a/crates/mind-body/src/self_run.rs +++ b/crates/mind-body/src/self_run.rs @@ -924,109 +924,9 @@ fn result_head(event: &serde_json::Map) -> String { head } -/// A result's words as they leave the actuator, for the journal and for her mind (the reply): redacted by `redact_url_userinfo`. +/// A result's words as they leave the actuator, for the journal and for her mind (the reply): redacted by `huginn_mind::redact::redact_url_userinfo`. fn result_text(event: &serde_json::Map) -> String { - redact_url_userinfo(event.get("result").and_then(Value::as_str).unwrap_or("")) -} - -/// The one redaction of URL credentials. Call it at each place a string the CLI produced enters the journal, her mind or chat: today -/// the Self result (`result_text`: journal line and Reply) and the structured output of a Persona or Sleep result -/// (`persona::structured_output`, string by string). It does not cover claude's stderr (inherited) or the CLI's transcript files. -/// -/// A forge token rides in a remote URL, so every `scheme://` and every protocol-relative `//` opens an authority, read as git, curl and -/// RFC 3986 read it: it ends at the first `/`, `?`, `#`, backslash, quote, backtick, parenthesis, bracket, angle bracket or ASCII whitespace -/// (`is_authority_end`). Its userinfo is everything before the authority's LAST `@` (a raw `@` in a password still redacts), and non-ASCII -/// characters, NBSP and U+2028 included, stay inside it. The userinfo becomes ``: `scheme://@host`. With a colon it is -/// always redacted; without one it is a bare token on http, https, ftp, ftps, ws, wss, any `+http`/`+https` scheme (`git+https`), protocol-relative `//` and an unparseable scheme, and a plain -/// username on any other scheme (`ssh://git@host`), which stays. A `@` after the authority (`github.com/@user`, `pkg@1.2.3`) is path, not -/// userinfo, and is left alone. Scheme-less credentials are covered only in the shape `word:secret@host.tld/` at the start of an -/// ASCII-whitespace-delimited word. Not covered: a password holding `/ ? # ` a quote or ASCII whitespace (no parser reads it as userinfo), -/// scp-style `user:secret@host:path`, tokens in query strings, claude's stderr, the kept transcripts. Syntax only; no reading of meaning. -pub(crate) fn redact_url_userinfo(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut rest = text; - while let Some(at) = rest.find("//") { - let (before, after) = rest.split_at(at); - let body = &after[2..]; - out.push_str(before); - out.push_str("//"); - // The scheme is the ASCII run before `:`. Without a colon, `//` opens an authority only at a word start (`a//b@c` is not one). - let scheme = match before.strip_suffix(':') { - // Only ASCII bytes match, so the byte after the last non-match is a character boundary. - Some(head) => head[head.bytes().rposition(|c| !(c.is_ascii_alphanumeric() || matches!(c, b'+' | b'-' | b'.'))).map_or(0, |i| i + 1)..].to_ascii_lowercase(), - None if before.chars().next_back().is_none_or(|c| is_break(c) || matches!(c, '"' | '\'' | '`' | '(' | '[' | '{' | '<' | '=' | ',')) => String::new(), - None => { - rest = body; - continue; - } - }; - let end = body.find(is_authority_end).unwrap_or(body.len()); - let authority = &body[..end]; - match authority.rfind('@') { - Some(i) if i > 0 && (authority[..i].contains(':') || carries_bare_token(&scheme)) => { - out.push_str(""); - out.push_str(&authority[i..]); - } - _ => out.push_str(authority), - } - rest = &body[end..]; - } - out.push_str(rest); - redact_schemeless(&out) -} - -/// Schemes whose lone userinfo is a token, not a username: web and ftp schemes, `ws`/`wss`, and `x+http`/`x+https` wrappers; `scheme` is lowercase. -fn carries_bare_token(scheme: &str) -> bool { - matches!(scheme, "" | "http" | "https" | "ftp" | "ftps" | "ws" | "wss") || scheme.ends_with("+http") || scheme.ends_with("+https") -} - -/// ASCII whitespace; NBSP, U+2028 and other Unicode whitespace are not, so a secret holding them is redacted whole. -fn is_break(c: char) -> bool { - matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c') -} - -/// The end of a URL's authority. -fn is_authority_end(c: char) -> bool { - is_break(c) || matches!(c, '/' | '?' | '#' | '\\' | '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | '<' | '>') -} - -/// `word:secret@host.tld/` at the start of a whitespace-delimited word that holds no `//` becomes `@host.tld/`. -fn redact_schemeless(text: &str) -> String { - let mut out = String::with_capacity(text.len()); - let mut word = String::new(); - let flush = |word: &mut String, out: &mut String| { - out.push_str(&redact_word(word)); - word.clear(); - }; - for c in text.chars() { - if is_break(c) { - flush(&mut word, &mut out); - out.push(c); - } else { - word.push(c); - } - } - flush(&mut word, &mut out); - out -} - -fn redact_word(word: &str) -> String { - let shaped = (|| { - if word.contains("//") { - return None; - } - let at = word.rfind('@')?; - let colon = word.find(':')?; - let (user, host) = (&word[..colon], &word[at + 1..]); - let host_name = &host[..host.find('/')?]; - let user_ok = !user.is_empty() && user.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')); - let host_ok = host_name.contains('.') && host_name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-')); - (colon + 1 < at && user_ok && host_ok).then_some(at) - })(); - match shaped { - Some(at) => format!("{}", &word[at..]), - None => word.to_string(), - } + huginn_mind::redact::redact_url_userinfo(event.get("result").and_then(Value::as_str).unwrap_or("")) } /// `text` in consecutive pieces of at most `limit` bytes, each cut at a character boundary; their concatenation is `text`. diff --git a/crates/mind-body/src/self_run/tests/reply.rs b/crates/mind-body/src/self_run/tests/reply.rs index 1e20e4d..acaf90d 100644 --- a/crates/mind-body/src/self_run/tests/reply.rs +++ b/crates/mind-body/src/self_run/tests/reply.rs @@ -122,18 +122,6 @@ fn a_refused_piece_of_a_long_result_holds_back_the_pieces_behind_it() { // A canary stands where a forge token would: no byte of it may reach the journal or her mind. const CANARY: &str = "CANARY-NOT-A-REAL-TOKEN-0000"; -#[test] -fn url_userinfo_is_redacted_where_it_is_a_secret_and_kept_where_it_is_a_username() { - let said = format!( - "a https://user:{CANARY}@host.example/r.git b https://{CANARY}@host.example/r.git c ssh://u:{CANARY}@h/x d http://{CANARY}@h e HTTPS://x:{CANARY}@h \ - f https://u:p@ss{CANARY}@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://u:{CANARY}@h) k ://@x" - ); - let want = "a https://@host.example/r.git b https://@host.example/r.git c ssh://@h/x d http://@h e HTTPS://@h \ - f https://@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://@h) k ://@x"; - assert_eq!(redact_url_userinfo(&said), want); - assert!(!redact_url_userinfo(&said).contains(CANARY)); -} - #[test] fn a_credential_in_a_url_reaches_neither_the_journal_nor_the_admitted_reply() { let mind = live_mind(); @@ -157,66 +145,6 @@ fn a_credential_split_by_the_reply_limit_is_redacted_before_the_cut() { assert!(!first.text.0.contains("CANARY") && !reply_of(&mind, 2).unwrap().text.0.contains("CANARY")); } -#[test] -fn a_secret_is_redacted_whole_by_the_authority_rule_and_commonplace_urls_are_left_alone() { - let r = redact_url_userinfo; - // Raw @, percent-encoding and non-ASCII (NBSP, U+2028) stay inside userinfo. - for secret in ["pa@ssT", "pa%40ssT", "pa\u{a0}ssT", "pa\u{2028}ssT", "pa:ssT", "\u{e9}\u{65e5}ssT"] { - let out = r(&format!("x https://u:{secret}@h/x")); - assert_eq!(out, "x https://@h/x", "{secret:?}"); - } - // The authority ends at / ? # \\ quotes, brackets and ASCII whitespace: no parser reads those as userinfo, so what follows is path. - assert_eq!(r("https://u:pa/ssT@h/x"), "https://u:pa/ssT@h/x"); - assert_eq!(r("https://u:pa ssT@h/x"), "https://u:pa ssT@h/x"); - assert_eq!(r("https://u:pa?ssT@h"), "https://u:pa?ssT@h"); - // Ordinary URLs with @ in the path are byte-identical. - for same in [ - "https://github.com/@user/repo", - "https://medium.com/@name/post-1", - "https://registry.npmjs.org/@scope/pkg", - "https://h/pkg@1.2.3", - "a//b@c", - "see //cdn.h/@x/y and https://h/?mail=a@b.c#f@g", - "https://h:8080/p git@github.com:org/r.git ssh://git@host/r.git ssh://git@h:22/r git://git@h/x", - ] { - assert_eq!(r(same), same); - } - // Compact JSON: only the credentialed URL changes. - let json = format!(r#"{{"a":"https://u:{CANARY}@h/x","b":"https://h2/@v","c":"https://h3/p@1"}}"#); - assert_eq!(r(&json), r#"{"a":"https://@h/x","b":"https://h2/@v","c":"https://h3/p@1"}"#); - assert_eq!(r("a/https://u:T@h/x,https://v:T@h2"), "a/https://@h/x,https://@h2"); - assert_eq!(r("HTTPS://TOKEN@h"), "HTTPS://@h"); - assert_eq!(r("[l](https://T@h/x) `https://u:T@h`"), "[l](https://@h/x) @h> `https://@h`"); - assert_eq!(r("a //u:T@host/x b //T@host c \"//T@h\" (//T@h)"), "a //@host/x b //@host c \"//@h\" (//@h)"); - // Token-bearing schemes beyond http: +http/+https wrappers and ws/wss redact a bare token; git+ssh and ssh keep their username. - assert_eq!(r("git+https://T@github.com/a/b GIT+HTTPS://T@h wss://T@h/ ws://T@h/"), "git+https://@github.com/a/b GIT+HTTPS://@h wss://@h/ ws://@h/"); - for same in ["git+ssh://git@host/x", "ssh://git@host/x"] { - assert_eq!(r(same), same); - } - // Scheme-less: only word:secret@host.tld/ . Each guard of the shape has its own case. - assert_eq!(r("user:T@host.tld/path"), "@host.tld/path"); - for same in ["a:b@c", "a:b@host/x", "a:b@ho_st.tld/x", "u$er:T@host.tld/x", "a:@host.tld/x", "mail me@x.org/ and :T@host.tld/x"] { - assert_eq!(r(same), same); - } -} - -#[test] -fn multibyte_text_never_panics_the_redaction() { - let r = redact_url_userinfo; - for case in ["see \u{201c}https://h/x", "\u{2192}https://h/x", "docs\u{ff1a}https://h/x", "\u{e9}://x", "\u{65e5}\u{672c}://u:T@h", "https://x/\u{e9}://", "\u{e9}//u:T@h", "\u{e9}:u:T@h.tld/"] { - let _ = r(case); - } - assert_eq!(r("\u{65e5}\u{672c}://u:T@h"), "\u{65e5}\u{672c}://@h"); - assert_eq!(r("\u{2192}https://u:T@h/x"), "\u{2192}https://@h/x"); - let seed = "\u{201c}\u{2192}\u{ff1a}\u{e9}\u{65e5}\u{a0}https://u:T@h/x//y:z@a.b/ ://@ //"; - let chars: Vec = seed.chars().collect(); - for start in 0..=chars.len() { - for end in start..=chars.len() { - let _ = r(&chars[start..end].iter().collect::()); - } - } -} - #[test] fn persona_and_sleep_structured_output_is_redacted_string_by_string() { let line = json!({"type": "result", "structured_output": {"say": format!("https://u:{CANARY}@h/x,"), "n": 3, "list": [format!("https://{CANARY}@h")]}}).to_string(); diff --git a/docs/chat-retrieval-map.md b/docs/chat-retrieval-map.md index f839cd4..bee6079 100644 --- a/docs/chat-retrieval-map.md +++ b/docs/chat-retrieval-map.md @@ -169,3 +169,32 @@ the cited message and its neighbours in the same room from this index, no embedder and no Synapse; an id the index lacks is refused naming `event_id`. The listing limit above is not changed by it: the cut spec does not assign that fix, so it stays open. + +## Status: ingest core (cut chat-ingest-core) + +`sources::chat::refresh_chat` writes chat points; scope is `sources/room_scope.rs` +and the credential scrub is `huginn_mind::redact`. Rules the code holds, which a +reader of the index must know: + +- A text edit changes the stored body in place, only when its sender is the + original's sender. Edits are never stored as events, so no payload field + records which edit changed a body. +- **Rebuild on an unknown redaction.** A redaction whose target is neither a + message in the room's stored chunks nor an event of the same page may be of an + edit whose own id Huginn never stored, so the body that edit set cannot be + taken back from the chunk. The room is then rebuilt: its points and cursor are + purged and the next read re-ingests its history from the port. Synapse serves a + redacted edit without its content (verified against Synapse 1.164.1), so the + original body returns and the edited text does not. Within one page, a redacted + edit is reverted in the fold without a rebuild. Redactions the rebuild itself + replays do not trigger another rebuild. A redaction of a media or state event + also lands here (the cost is one rebuild of that room). +- **Edits of a redacted original are dropped.** Synapse keeps serving the + edits of a redacted original, with their replacement text, so Huginn drops any + edit whose target is not a message it holds or has just read (a redacted, + purged, never-stored or unknown original), live or late, in the fold and in a + rebuild alike. +- How the Matrix port maps a pruned event (`content: {}`) to `EventKind::Other` + is the ingest-job cut's to pin; this core only assumes it does. +- An error reading a joined room's facts is not a reclassification: the room + keeps its points and cursor, and the run's report for the room says why.