chat ingest core: room scope, credential scrub, chunker, refresh_chat, steward purge (#60)
This commit is contained in:
parent
f1d5c4039b
commit
b4f3a48a8f
16 changed files with 2840 additions and 192 deletions
|
|
@ -1303,6 +1303,22 @@ impl huginn_daemon::sources::SourceIndex for Desk {
|
|||
anyhow::bail!("a daemon never writes the source index")
|
||||
}
|
||||
|
||||
fn chat_tail(&mut self, _room_id: &str) -> anyhow::Result<Option<huginn_daemon::sources::StoredChat>> {
|
||||
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||
}
|
||||
|
||||
fn chat_chunks_holding(&mut self, _room_id: &str, _event_ids: &[String]) -> anyhow::Result<Vec<huginn_daemon::sources::StoredChat>> {
|
||||
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||
}
|
||||
|
||||
fn chat_chunks_overlapping(&mut self, _room_id: &str, _from_ms: u64, _to_ms: u64) -> anyhow::Result<Vec<huginn_daemon::sources::StoredChat>> {
|
||||
anyhow::bail!("a daemon never reads the ingest job's side of the source index")
|
||||
}
|
||||
|
||||
fn delete_chat(&mut self, _room_id: &str, _ids: Option<&[String]>) -> anyhow::Result<()> {
|
||||
anyhow::bail!("a daemon never writes the source index")
|
||||
}
|
||||
|
||||
fn chat_nearest(
|
||||
&mut self,
|
||||
_vector: &[f32],
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use std::time::Duration;
|
|||
|
||||
use anyhow::{Result, bail, ensure};
|
||||
|
||||
use crate::sources::{ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta};
|
||||
use crate::sources::{ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta, StoredChat};
|
||||
|
||||
use super::{CollectionMeta, Described, Embedder, Hit, ModelIdentity, Point, VectorIndex, point_id};
|
||||
|
||||
|
|
@ -264,6 +264,16 @@ pub(crate) struct SourcesState {
|
|||
pub(crate) deletes: Vec<(String, Option<Vec<String>>)>,
|
||||
/// A delete naming this repo refuses.
|
||||
pub(crate) fail_deletes_of: Option<String>,
|
||||
/// Every cursor write refuses once this many have been written.
|
||||
pub(crate) fail_cursor_writes_after: Option<usize>,
|
||||
/// Every chat upsert refuses while this is set.
|
||||
pub(crate) fail_chat_upserts: bool,
|
||||
/// Every chat delete refuses while this is set.
|
||||
pub(crate) fail_chat_deletes: bool,
|
||||
/// A chat delete naming one of these rooms refuses.
|
||||
pub(crate) fail_chat_deletes_for: BTreeSet<String>,
|
||||
/// Every chat delete that went through: the room and the ids (`None` is all of the room).
|
||||
pub(crate) chat_deletes: Vec<(String, Option<Vec<String>>)>,
|
||||
/// The repo filter and limit of every search, in order.
|
||||
pub(crate) searches: Vec<(Option<String>, u32)>,
|
||||
}
|
||||
|
|
@ -326,6 +336,9 @@ impl SourceIndex for FakeSources {
|
|||
|
||||
fn set_cursors(&mut self, cursors: &BTreeMap<String, String>) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if state.fail_cursor_writes_after.is_some_and(|after| state.cursor_writes.len() >= after) {
|
||||
bail!("the vector store refused the cursor write");
|
||||
}
|
||||
state.cursor_writes.push(cursors.clone());
|
||||
let Some(DescribedSources::Labelled { cursors: held, .. }) = &mut state.described else { bail!("no source collection") };
|
||||
*held = cursors.clone();
|
||||
|
|
@ -375,6 +388,7 @@ impl SourceIndex for FakeSources {
|
|||
|
||||
fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
ensure!(!state.fail_chat_upserts, "the vector store refused the chat upsert");
|
||||
for point in points {
|
||||
ensure!(!point.payload.messages.is_empty(), "a chat chunk carries no messages");
|
||||
state.chat.insert(point.id.clone(), point.clone());
|
||||
|
|
@ -404,6 +418,44 @@ impl SourceIndex for FakeSources {
|
|||
}
|
||||
Ok(kept.into_iter().take(limit as usize).cloned().collect())
|
||||
}
|
||||
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id).max_by_key(|(id, point)| (span(&point.payload).1, (*id).clone())).map(stored))
|
||||
}
|
||||
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
Ok(state
|
||||
.chat
|
||||
.iter()
|
||||
.filter(|(_, point)| point.payload.room_id == room_id && point.payload.messages.iter().any(|message| event_ids.contains(&message.event_id)))
|
||||
.map(stored)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||
let state = self.state.lock().unwrap();
|
||||
Ok(state.chat.iter().filter(|(_, point)| point.payload.room_id == room_id && span(&point.payload).1 >= from_ms && span(&point.payload).0 < to_ms).map(stored).collect())
|
||||
}
|
||||
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
ensure!(!state.fail_chat_deletes && !state.fail_chat_deletes_for.contains(room_id), "the vector store refused the chat delete");
|
||||
state.chat_deletes.push((room_id.to_owned(), ids.map(<[String]>::to_vec)));
|
||||
state.chat.retain(|id, point| !(point.payload.room_id == room_id && ids.is_none_or(|ids| ids.contains(id))));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn stored((id, point): (&String, &ChatPoint)) -> StoredChat {
|
||||
StoredChat { id: id.clone(), payload: point.payload.clone() }
|
||||
}
|
||||
|
||||
/// (start, end) of the times a chunk's messages make.
|
||||
fn span(payload: &ChatPayload) -> (u64, u64) {
|
||||
let times = || payload.messages.iter().map(|message| message.ts_ms);
|
||||
(times().min().unwrap_or(0), times().max().unwrap_or(0))
|
||||
}
|
||||
|
||||
/// A chat chunk under a filter, by the span its messages make.
|
||||
|
|
@ -443,4 +495,16 @@ impl SourceIndex for FakeIndex {
|
|||
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>> {
|
||||
self.sources.chat_scroll(filter, order, limit, within)
|
||||
}
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||
self.sources.chat_tail(room_id)
|
||||
}
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||
self.sources.chat_chunks_holding(room_id, event_ids)
|
||||
}
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||
self.sources.chat_chunks_overlapping(room_id, from_ms, to_ms)
|
||||
}
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||
self.sources.delete_chat(room_id, ids)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use huginn_mind::ChatMessage;
|
|||
|
||||
use crate::sources::{
|
||||
COLLECTION, CORPUS_CHAT, CORPUS_FORGE, ChatFilter, ChatOrder, ChatPayload, ChatPoint, ChatRow, DescribedSources, FRONT_PAGE_FILES, SourceIndex, SourcePayload, SourcePoint, SourceRow,
|
||||
SourcesMeta,
|
||||
SourcesMeta, StoredChat,
|
||||
};
|
||||
|
||||
use super::{CollectionMeta, INDEX_CALL_TIMEOUT, Described, Hit, Http, Method, Point, VectorIndex};
|
||||
|
|
@ -464,6 +464,74 @@ impl SourceIndex for QdrantIndex {
|
|||
let found = answer["result"]["points"].as_array().context("Qdrant's scroll has no points")?;
|
||||
found.iter().map(|point| chat_payload_of(&point["payload"])).collect()
|
||||
}
|
||||
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>> {
|
||||
let order = json!({ "key": "ts_end_ms", "direction": "desc" });
|
||||
let Some(newest) = self.scroll_chat(chat_room_filter(room_id, vec![]), Some((order, 1)))?.into_iter().next() else { return Ok(None) };
|
||||
// Chunks tied on the greatest end are all read and the greatest id wins, so the tail is the same chunk on every run.
|
||||
let end = newest.payload.messages.iter().map(|message| message.ts_ms).max().unwrap_or(0);
|
||||
let tied = self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "ts_end_ms", "range": { "gte": end } })]), None)?;
|
||||
Ok(tied.into_iter().max_by(|a, b| a.id.cmp(&b.id)))
|
||||
}
|
||||
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>> {
|
||||
if event_ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.scroll_chat(chat_room_filter(room_id, vec![json!({ "key": "event_ids", "match": { "any": event_ids } })]), None)
|
||||
}
|
||||
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>> {
|
||||
let span = vec![json!({ "key": "ts_end_ms", "range": { "gte": from_ms } }), json!({ "key": "ts_start_ms", "range": { "lt": to_ms } })];
|
||||
self.scroll_chat(chat_room_filter(room_id, span), None)
|
||||
}
|
||||
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()> {
|
||||
let collection = self.sources.clone();
|
||||
let body = json!({ "filter": chat_delete_filter(room_id, ids) });
|
||||
self.ask(Method::Post, &format!("/collections/{collection}/points/delete?wait=true"), Some(&body), &[200])?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl QdrantIndex {
|
||||
/// Every chat chunk `filter` keeps, with its point id, paged; or, given an
|
||||
/// order and a limit, the first of them in that order.
|
||||
fn scroll_chat(&mut self, filter: Value, ordered: Option<(Value, u64)>) -> Result<Vec<StoredChat>> {
|
||||
let collection = self.sources.clone();
|
||||
let mut out = Vec::new();
|
||||
let mut offset = Value::Null;
|
||||
loop {
|
||||
let mut body = json!({ "filter": filter, "limit": SCROLL_PAGE, "with_payload": true, "with_vector": false });
|
||||
if let Some((order, limit)) = &ordered {
|
||||
body["order_by"] = order.clone();
|
||||
body["limit"] = json!(limit);
|
||||
} else if !offset.is_null() {
|
||||
body["offset"] = offset;
|
||||
}
|
||||
let (_, answer) = self.ask(Method::Post, &format!("/collections/{collection}/points/scroll"), Some(&body), &[200])?;
|
||||
for point in answer["result"]["points"].as_array().context("Qdrant's scroll has no points")? {
|
||||
out.push(StoredChat { id: point["id"].as_str().context("a point id is not a string")?.to_owned(), payload: chat_payload_of(&point["payload"])? });
|
||||
}
|
||||
offset = answer["result"]["next_page_offset"].clone();
|
||||
if ordered.is_some() || offset.is_null() {
|
||||
return Ok(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// What a chat delete removes: the room's chat points, all of them or those with `ids`. The corpus
|
||||
/// and room conditions hold either way, so a forge point or another room's point cannot match.
|
||||
fn chat_delete_filter(room_id: &str, ids: Option<&[String]>) -> Value {
|
||||
chat_room_filter(room_id, ids.map(|ids| json!({ "has_id": ids })).into_iter().collect())
|
||||
}
|
||||
|
||||
/// The chat points of one room, narrowed by `more` conditions.
|
||||
fn chat_room_filter(room_id: &str, more: Vec<Value>) -> Value {
|
||||
let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_CHAT } }), json!({ "key": "room_id", "match": { "value": room_id } })];
|
||||
must.extend(more);
|
||||
json!({ "must": must })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -526,6 +594,29 @@ mod source_filter_tests {
|
|||
assert!(chat_payload_of(&forge).is_err());
|
||||
}
|
||||
|
||||
/// The writer-side filters name the chat corpus and the room, so a forge point never matches.
|
||||
#[test]
|
||||
fn the_writer_filters_are_always_chat_corpus_and_the_room() {
|
||||
let filter = chat_room_filter("!r:x", vec![json!({ "key": "event_ids", "match": { "any": ["$a"] } })]);
|
||||
assert_eq!(
|
||||
filter,
|
||||
json!({ "must": [
|
||||
{ "key": "corpus", "match": { "value": "chat" } },
|
||||
{ "key": "room_id", "match": { "value": "!r:x" } },
|
||||
{ "key": "event_ids", "match": { "any": ["$a"] } },
|
||||
] })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chat_delete_names_the_corpus_and_room_with_or_without_ids() {
|
||||
let all = chat_delete_filter("!r:x", None);
|
||||
assert_eq!(all["must"].as_array().unwrap().len(), 2);
|
||||
let some = chat_delete_filter("!r:x", Some(&["p1".to_owned()]));
|
||||
assert_eq!(some["must"][2], json!({ "has_id": ["p1"] }));
|
||||
assert_eq!(some["must"][0], json!({ "key": "corpus", "match": { "value": "chat" } }));
|
||||
}
|
||||
|
||||
/// The chat filter is the chat corpus, the rooms as an any-of `should` split
|
||||
/// by id and alias, and the two ranges that keep an overlapping span.
|
||||
#[test]
|
||||
|
|
|
|||
638
crates/huginn-daemon/src/sources/chat.rs
Normal file
638
crates/huginn-daemon/src/sources/chat.rs
Normal file
|
|
@ -0,0 +1,638 @@
|
|||
//! Chat ingest: joined-room history into the shared source collection, beside
|
||||
//! the forge's repos. `refresh_chat` is the only writer of `corpus=chat` points
|
||||
//! and of the cursors object's `chat:` keys; `chat_purge::request_purge` is the
|
||||
//! only writer of `chat-purge:` keys, and `refresh_chat` the only applier.
|
||||
//!
|
||||
//! The Matrix client that implements `RoomHistory` is the next cut's and holds no
|
||||
//! policy. What is in scope is `room_scope`'s; what may be embedded is
|
||||
//! `huginn_mind::redact::scrub`'s; this file folds events into chunks and writes
|
||||
//! them, write-then-delete and cursor-last, so any crash replays to the same state.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use huginn_mind::ChatMessage;
|
||||
use huginn_mind::redact::{ScrubKind, scrub};
|
||||
|
||||
use super::chat_chunk::{chat_version, chunk_id, chunk_messages, chunk_payload, cut_body};
|
||||
use super::chat_purge::{PurgeWindow, covers, parse_windows, purge_key};
|
||||
use super::room_scope::{Estate, Refusal, RoomFacts, admit_room, refuse_event};
|
||||
use super::{CORPUS, ChatPoint, DescribedSources, SourceIndex, StoredChat};
|
||||
use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Embedder, INDEX_CALL_TIMEOUT};
|
||||
|
||||
/// Events asked of the port per page.
|
||||
pub const CHAT_PAGE: u32 = 200;
|
||||
|
||||
/// A chat cursor is the key `chat:<room_id>`; a forge repo name cannot contain
|
||||
/// a colon. The only definition of the prefix.
|
||||
pub const CHAT_CURSOR_PREFIX: &str = "chat:";
|
||||
|
||||
/// A steward's purge tombstone is the key `chat-purge:<room_id>`. The only
|
||||
/// definition of the prefix.
|
||||
pub const CHAT_PURGE_PREFIX: &str = "chat-purge:";
|
||||
|
||||
/// Whether a cursors-object key is chat's (a room cursor or a purge tombstone)
|
||||
/// and not a forge repo name. The only test that tells the two apart.
|
||||
pub fn is_chat_key(key: &str) -> bool {
|
||||
key.starts_with(CHAT_CURSOR_PREFIX) || key.starts_with(CHAT_PURGE_PREFIX)
|
||||
}
|
||||
|
||||
/// What one room event is to the ingest. Only text, notice and emote messages
|
||||
/// become `Message`; everything else (encrypted, media, state, server-redacted)
|
||||
/// is `Other` and is never stored.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum EventKind {
|
||||
Message { body: String },
|
||||
/// A replacement of `target`'s body.
|
||||
Edit { target: String, body: String },
|
||||
Redaction { target: String },
|
||||
Other,
|
||||
}
|
||||
|
||||
/// One event of a room, as the port hands it over: the Matrix client maps its
|
||||
/// JSON to this and filters nothing. `content_keys` are the top-level keys of
|
||||
/// the event's content (the scope reads them).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RoomEvent {
|
||||
pub event_id: String,
|
||||
pub sender: String,
|
||||
pub ts_ms: u64,
|
||||
pub content_keys: Vec<String>,
|
||||
pub kind: EventKind,
|
||||
}
|
||||
|
||||
/// One page of a room's events after a cursor event, oldest first.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct EventPage {
|
||||
pub events: Vec<RoomEvent>,
|
||||
/// More events follow this page.
|
||||
pub more: bool,
|
||||
}
|
||||
|
||||
/// The reader user's view of Synapse as ingest uses it. Holds no policy: it maps
|
||||
/// the server's answers to these types and filters nothing.
|
||||
pub trait RoomHistory {
|
||||
/// The rooms the reader has joined. The only source of rooms; there is no allow list.
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>>;
|
||||
fn facts(&mut self, room_id: &str) -> Result<RoomFacts>;
|
||||
/// Up to `limit` events after the event `after` (from the room's start when `None`).
|
||||
/// `Ok(None)` means `after` is unknown to the server.
|
||||
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>>;
|
||||
}
|
||||
|
||||
/// The cursors-object key of a room's chat cursor.
|
||||
pub fn cursor_key(room_id: &str) -> String {
|
||||
format!("{CHAT_CURSOR_PREFIX}{room_id}")
|
||||
}
|
||||
|
||||
/// The room a chat cursor key belongs to; `None` for any other key (a purge tombstone included).
|
||||
pub fn cursor_room(key: &str) -> Option<&str> {
|
||||
key.strip_prefix(CHAT_CURSOR_PREFIX)
|
||||
}
|
||||
|
||||
/// A cursor value, `<version>:<last event id>`; the event is empty before a room's first upsert.
|
||||
pub fn parse_cursor(value: &str) -> Option<(&str, &str)> {
|
||||
value.split_once(':')
|
||||
}
|
||||
|
||||
/// What one admitted room's run did, for the journal. Never carries a body or a sender.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ChatReport {
|
||||
pub room_id: String,
|
||||
pub alias: Option<String>,
|
||||
pub first: Option<String>,
|
||||
pub last: Option<String>,
|
||||
pub events: u32,
|
||||
pub chunks: u32,
|
||||
pub scrubbed: u32,
|
||||
pub purged: u32,
|
||||
pub seconds: f64,
|
||||
/// Why the room could not be read or written this run; what the index held for it stays as it was.
|
||||
pub failure: Option<String>,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ChatReport {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
formatter,
|
||||
"{} ({}) {}..{} events={} chunks={} scrubbed={} purged={} seconds={:.1}",
|
||||
self.alias.as_deref().unwrap_or("-"),
|
||||
self.room_id,
|
||||
self.first.as_deref().unwrap_or("-"),
|
||||
self.last.as_deref().unwrap_or("-"),
|
||||
self.events,
|
||||
self.chunks,
|
||||
self.scrubbed,
|
||||
self.purged,
|
||||
self.seconds
|
||||
)?;
|
||||
match &self.failure {
|
||||
Some(why) => write!(formatter, " failed: {why}"),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A joined room the scope refused, whole, and why; `event_id` when an event did it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RefusedRoom {
|
||||
pub room_id: String,
|
||||
pub reason: Refusal,
|
||||
pub event_id: Option<String>,
|
||||
}
|
||||
|
||||
/// A credential the scrub replaced: where, never what.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScrubHit {
|
||||
pub room_id: String,
|
||||
pub event_id: String,
|
||||
pub kind: ScrubKind,
|
||||
}
|
||||
|
||||
/// What a chat refresh did.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct ChatRefreshed {
|
||||
pub rooms: Vec<ChatReport>,
|
||||
/// Rooms (or steps) that failed; the others went on.
|
||||
pub failed: Vec<String>,
|
||||
pub refused: Vec<RefusedRoom>,
|
||||
pub scrubbed: Vec<ScrubHit>,
|
||||
}
|
||||
|
||||
impl ChatReport {
|
||||
fn empty(room_id: &str, alias: Option<String>) -> Self {
|
||||
Self { room_id: room_id.to_owned(), alias, first: None, last: None, events: 0, chunks: 0, scrubbed: 0, purged: 0, seconds: 0.0, failure: None }
|
||||
}
|
||||
|
||||
/// A room that could not be handled this run. Its points and cursor are untouched: an error is not a reclassification.
|
||||
fn failed(room_id: &str, alias: Option<String>, error: &anyhow::Error) -> Self {
|
||||
Self { failure: Some(format!("{error:#}")), ..Self::empty(room_id, alias) }
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatRefreshed {
|
||||
/// Nothing failed and no joined room had to be refused: a joined estate, direct,
|
||||
/// encrypted or bridged room means the reader's roster is wrong. A scrub hit does not make a run unclean.
|
||||
pub fn is_clean(&self) -> bool {
|
||||
self.failed.is_empty() && self.refused.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
fn read_cursors<S: SourceIndex>(index: &mut S) -> Result<BTreeMap<String, String>> {
|
||||
match index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? {
|
||||
DescribedSources::Labelled { cursors, .. } => Ok(cursors),
|
||||
_ => bail!("the source collection is gone; the forge refresh creates it"),
|
||||
}
|
||||
}
|
||||
|
||||
fn put_cursor<S: SourceIndex>(index: &mut S, key: &str, value: &str) -> Result<()> {
|
||||
let mut cursors = read_cursors(index)?;
|
||||
if cursors.insert(key.to_owned(), value.to_owned()).as_deref() != Some(value) {
|
||||
index.set_cursors(&cursors).context("writing a chat cursor")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn drop_cursor<S: SourceIndex>(index: &mut S, key: &str) -> Result<()> {
|
||||
let mut cursors = read_cursors(index)?;
|
||||
if cursors.remove(key).is_some() {
|
||||
index.set_cursors(&cursors).context("dropping a chat cursor")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A room's points, then its `chat:` cursor. Its purge tombstone stays.
|
||||
fn purge_room<S: SourceIndex>(index: &mut S, room_id: &str) -> Result<()> {
|
||||
index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id}"))?;
|
||||
drop_cursor(index, &cursor_key(room_id))
|
||||
}
|
||||
|
||||
/// What `retire_chat` did: the rooms it removed, and those it could not (each with why), which it went on past.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct Retired {
|
||||
pub retired: Vec<String>,
|
||||
pub failed: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Deletes the points and `chat:` cursor of every room not in `keep` (the reader left it,
|
||||
/// or it is refused or excluded). A room whose delete fails is reported and the others go on. Purge
|
||||
/// tombstones and forge repos are never touched. With the empty set it removes the whole chat corpus.
|
||||
pub fn retire_chat<S: SourceIndex>(index: &mut S, keep: &BTreeSet<String>) -> Result<Retired> {
|
||||
let rooms: Vec<String> = read_cursors(index)?.keys().filter_map(|key| cursor_room(key)).filter(|room| !keep.contains(*room)).map(str::to_owned).collect();
|
||||
let mut done = Retired::default();
|
||||
for room in rooms {
|
||||
match purge_room(index, &room) {
|
||||
Ok(()) => done.retired.push(room),
|
||||
Err(error) => done.failed.push((room, format!("{error:#}"))),
|
||||
}
|
||||
}
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
/// A change to chunks already stored, in event order.
|
||||
enum Op {
|
||||
/// An edit by `sender` of `target`'s body; applies only to the original's sender.
|
||||
Edit { id: String, target: String, sender: String, body: String },
|
||||
/// A held message's body is what the page says it is (a replay).
|
||||
Set { target: String, body: String },
|
||||
Redact { target: String },
|
||||
}
|
||||
|
||||
impl Op {
|
||||
fn target(&self) -> &str {
|
||||
match self {
|
||||
Self::Edit { target, .. } | Self::Set { target, .. } | Self::Redact { target } => target,
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies to `messages`; whether they changed.
|
||||
fn apply(&self, messages: &mut Vec<ChatMessage>) -> bool {
|
||||
let Some(at) = messages.iter().position(|message| message.event_id == self.target()) else { return false };
|
||||
match self {
|
||||
Self::Edit { sender, body, .. } if messages[at].sender == *sender && messages[at].body != *body => messages[at].body.clone_from(body),
|
||||
Self::Set { body, .. } if messages[at].body != *body => messages[at].body.clone_from(body),
|
||||
Self::Redact { .. } => {
|
||||
messages.remove(at);
|
||||
}
|
||||
_ => return false,
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// A message of the page with the edits of the page that apply to it: the body is the last edit's, and a
|
||||
/// redacted edit leaves the list, so the body falls back to the one before it.
|
||||
struct Pending {
|
||||
message: ChatMessage,
|
||||
original: String,
|
||||
edits: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
struct Folded {
|
||||
pending: Vec<ChatMessage>,
|
||||
ops: Vec<Op>,
|
||||
/// Every event id of the page: a redaction of one of them is known.
|
||||
seen: HashSet<String>,
|
||||
}
|
||||
|
||||
struct Room<'a> {
|
||||
facts: &'a RoomFacts,
|
||||
label: String,
|
||||
windows: Vec<PurgeWindow>,
|
||||
}
|
||||
|
||||
/// The page folded in order: the purge filter, then the credential scrub, then the
|
||||
/// event's effect. After this no raw body exists.
|
||||
fn fold(room: &Room, events: &[RoomEvent], report: &mut ChatReport, out: &mut ChatRefreshed) -> Folded {
|
||||
let mut pending: Vec<Pending> = Vec::new();
|
||||
let mut ops: Vec<Op> = Vec::new();
|
||||
let mut scrubbed = |event: &RoomEvent, body: &str| {
|
||||
let clean = scrub(body);
|
||||
for kind in clean.hits {
|
||||
report.scrubbed += 1;
|
||||
out.scrubbed.push(ScrubHit { room_id: room.facts.room_id.clone(), event_id: event.event_id.clone(), kind });
|
||||
}
|
||||
cut_body(&clean.text)
|
||||
};
|
||||
for event in events {
|
||||
match &event.kind {
|
||||
EventKind::Message { body } => {
|
||||
if covers(&room.windows, event.ts_ms) {
|
||||
report.purged += 1;
|
||||
} else {
|
||||
let body = scrubbed(event, body);
|
||||
if !body.trim().is_empty() {
|
||||
let message = ChatMessage { event_id: event.event_id.clone(), sender: event.sender.clone(), ts_ms: event.ts_ms, body: body.clone() };
|
||||
pending.push(Pending { message, original: body, edits: Vec::new() });
|
||||
}
|
||||
}
|
||||
}
|
||||
EventKind::Edit { target, body } => {
|
||||
if covers(&room.windows, event.ts_ms) {
|
||||
report.purged += 1;
|
||||
continue;
|
||||
}
|
||||
let body = scrubbed(event, body);
|
||||
match pending.iter_mut().find(|held| held.message.event_id == *target) {
|
||||
Some(held) if held.message.sender == event.sender && !body.trim().is_empty() => {
|
||||
held.edits.push((event.event_id.clone(), body.clone()));
|
||||
held.message.body = body;
|
||||
}
|
||||
Some(_) => {}
|
||||
None => ops.push(Op::Edit { id: event.event_id.clone(), target: target.clone(), sender: event.sender.clone(), body }),
|
||||
}
|
||||
}
|
||||
EventKind::Redaction { target } => {
|
||||
// The target may be a message, or an edit whose own text must stop applying.
|
||||
pending.retain(|held| held.message.event_id != *target);
|
||||
for held in &mut pending {
|
||||
held.edits.retain(|(id, _)| id != target);
|
||||
held.message.body = held.edits.last().map_or_else(|| held.original.clone(), |(_, body)| body.clone());
|
||||
}
|
||||
ops.retain(|op| !matches!(op, Op::Edit { id, .. } if id == target));
|
||||
ops.push(Op::Redact { target: target.clone() });
|
||||
}
|
||||
EventKind::Other => {}
|
||||
}
|
||||
}
|
||||
Folded { pending: pending.into_iter().map(|held| held.message).collect(), ops, seen: events.iter().map(|event| event.event_id.clone()).collect() }
|
||||
}
|
||||
|
||||
/// Chunks to rewrite: the stored chunk ids they replace and the messages that now make them.
|
||||
type Rewrite = (Vec<String>, Vec<ChatMessage>);
|
||||
|
||||
/// Chunks `rewrites`, embeds and writes the new chunks (acknowledged), and only then deletes the old
|
||||
/// ids that are not among the new. Returns how many chunks were written.
|
||||
///
|
||||
/// A chunk written by an earlier run whose delete failed can still sit beside the chunk that replaced
|
||||
/// it, and a rewrite of the stale one then names the same first event, hence the same id, as the
|
||||
/// newer chunk. The newer chunk (`standing`, the other chunks the page found) stays; the stale one is deleted.
|
||||
fn write_rewrites<E: Embedder, S: SourceIndex>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
dimensions: u32,
|
||||
room: &Room,
|
||||
rewrites: Vec<Rewrite>,
|
||||
standing: &BTreeSet<String>,
|
||||
) -> Result<u32> {
|
||||
let mut old: BTreeSet<String> = BTreeSet::new();
|
||||
let mut points: Vec<(String, super::ChatPayload)> = Vec::new();
|
||||
for (ids, messages) in rewrites {
|
||||
for group in chunk_messages(&room.label, &messages) {
|
||||
let id = chunk_id(&room.facts.room_id, &group[0].event_id);
|
||||
if standing.contains(&id) && !ids.contains(&id) {
|
||||
continue;
|
||||
}
|
||||
points.push((id, chunk_payload(&room.facts.room_id, room.facts.alias.as_deref(), &room.label, group)));
|
||||
}
|
||||
old.extend(ids);
|
||||
}
|
||||
let new: BTreeSet<&String> = points.iter().map(|(id, _)| id).collect();
|
||||
let stale: Vec<String> = old.iter().filter(|id| !new.contains(id)).cloned().collect();
|
||||
let written = u32::try_from(points.len()).unwrap_or(u32::MAX);
|
||||
for slice in points.chunks(BATCH) {
|
||||
let texts: Vec<String> = slice.iter().map(|(_, payload)| payload.text.clone()).collect();
|
||||
let vectors = embedder.embed(&texts, BULK_EMBED_TIMEOUT).context("embedding chat chunks")?;
|
||||
ensure!(vectors.len() == texts.len(), "the embedder returned {} vectors for {} chunks", vectors.len(), texts.len());
|
||||
let mut batch = Vec::with_capacity(slice.len());
|
||||
for ((id, payload), vector) in slice.iter().zip(vectors) {
|
||||
ensure!(vector.len() == dimensions as usize, "the embedder returned a vector of {} where the collection holds {dimensions}", vector.len());
|
||||
batch.push(ChatPoint { id: id.clone(), vector, payload: payload.clone() });
|
||||
}
|
||||
index.upsert_chat(&batch).context("writing chat chunks")?;
|
||||
}
|
||||
if !stale.is_empty() {
|
||||
index.delete_chat(&room.facts.room_id, Some(&stale)).context("deleting superseded chat chunks")?;
|
||||
}
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
/// Applies the room's purge windows to the chunks it already holds.
|
||||
fn apply_purge<E: Embedder, S: SourceIndex>(embedder: &mut E, index: &mut S, dimensions: u32, room: &Room, report: &mut ChatReport) -> Result<()> {
|
||||
let mut hit: BTreeMap<String, StoredChat> = BTreeMap::new();
|
||||
for window in &room.windows {
|
||||
for chunk in index.chat_chunks_overlapping(&room.facts.room_id, window.from_ms, window.to_ms)? {
|
||||
hit.insert(chunk.id.clone(), chunk);
|
||||
}
|
||||
}
|
||||
let mut rewrites = Vec::new();
|
||||
for chunk in hit.into_values() {
|
||||
let kept: Vec<ChatMessage> = chunk.payload.messages.iter().filter(|message| !covers(&room.windows, message.ts_ms)).cloned().collect();
|
||||
let removed = chunk.payload.messages.len() - kept.len();
|
||||
if removed > 0 {
|
||||
report.purged += u32::try_from(removed).unwrap_or(u32::MAX);
|
||||
rewrites.push((vec![chunk.id], kept));
|
||||
}
|
||||
}
|
||||
report.chunks += write_rewrites(embedder, index, dimensions, room, rewrites, &BTreeSet::new())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Folds one page into the stored chunks: edits and redactions rewrite the chunk that held
|
||||
/// the target alone, new messages extend the open tail.
|
||||
fn write_page<E: Embedder, S: SourceIndex>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
dimensions: u32,
|
||||
room: &Room,
|
||||
mut folded: Folded,
|
||||
may_rebuild: bool,
|
||||
) -> Result<Option<u32>> {
|
||||
let room_id = &room.facts.room_id;
|
||||
let mut asked: Vec<String> = folded.pending.iter().map(|message| message.event_id.clone()).collect();
|
||||
asked.extend(folded.ops.iter().map(|op| op.target().to_owned()));
|
||||
if asked.is_empty() {
|
||||
return Ok(Some(0));
|
||||
}
|
||||
let holding = index.chat_chunks_holding(room_id, &asked)?;
|
||||
// A message the index already holds is a replay: its page body is asserted, never appended again.
|
||||
let held: HashSet<&str> = holding.iter().flat_map(|chunk| chunk.payload.messages.iter().map(|message| message.event_id.as_str())).collect();
|
||||
// A redaction of an event this room neither holds nor shows on the page may be of an edit whose own id is
|
||||
// never stored, so a body it changed cannot be taken back here: the room is rebuilt from the history, which
|
||||
// serves a redacted edit without its text.
|
||||
if may_rebuild && folded.ops.iter().any(|op| matches!(op, Op::Redact { target } if !folded.seen.contains(target) && !held.contains(target.as_str()))) {
|
||||
return Ok(None);
|
||||
}
|
||||
let (replayed, fresh): (Vec<ChatMessage>, Vec<ChatMessage>) = folded.pending.drain(..).partition(|message| held.contains(message.event_id.as_str()));
|
||||
let mut ops: Vec<Op> = replayed.into_iter().map(|message| Op::Set { target: message.event_id, body: message.body }).collect();
|
||||
ops.append(&mut folded.ops);
|
||||
|
||||
let tail = index.chat_tail(room_id)?;
|
||||
let tail_id = tail.as_ref().map(|tail| tail.id.clone());
|
||||
let standing: BTreeSet<String> = holding.iter().map(|chunk| chunk.id.clone()).chain(tail_id.clone()).collect();
|
||||
let mut tail_messages = tail.map(|tail| tail.payload.messages).unwrap_or_default();
|
||||
let mut tail_changed = false;
|
||||
let mut rewrites: Vec<Rewrite> = Vec::new();
|
||||
for chunk in holding {
|
||||
let mut messages = chunk.payload.messages.clone();
|
||||
let mut changed = false;
|
||||
for op in &ops {
|
||||
changed |= op.apply(&mut messages);
|
||||
}
|
||||
if Some(&chunk.id) == tail_id.as_ref() {
|
||||
(tail_messages, tail_changed) = (messages, changed);
|
||||
} else if changed {
|
||||
rewrites.push((vec![chunk.id], messages));
|
||||
}
|
||||
}
|
||||
if !fresh.is_empty() || tail_changed {
|
||||
tail_messages.extend(fresh);
|
||||
rewrites.push((tail_id.into_iter().collect(), tail_messages));
|
||||
}
|
||||
write_rewrites(embedder, index, dimensions, room, rewrites, &standing).map(Some)
|
||||
}
|
||||
|
||||
enum Outcome {
|
||||
Done(ChatReport),
|
||||
Refused(RefusedRoom),
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn ingest_room<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
history: &mut H,
|
||||
dimensions: u32,
|
||||
facts: &RoomFacts,
|
||||
out: &mut ChatRefreshed,
|
||||
) -> Result<Outcome> {
|
||||
let started = Instant::now();
|
||||
let room_id = facts.room_id.as_str();
|
||||
let (key, version) = (cursor_key(room_id), chat_version());
|
||||
let cursors = read_cursors(index)?;
|
||||
let mut may_rebuild = false;
|
||||
let mut after: Option<String> = match cursors.get(&key).map(|value| parse_cursor(value)) {
|
||||
None => None,
|
||||
Some(Some((held, event))) if held == version => {
|
||||
may_rebuild = true;
|
||||
Some(event.to_owned()).filter(|event| !event.is_empty())
|
||||
}
|
||||
// Another chunker or scrub version, or a value nobody here wrote: this room alone starts over.
|
||||
Some(_) => {
|
||||
index.delete_chat(room_id, None).with_context(|| format!("deleting the chunks of {room_id} for a re-ingest"))?;
|
||||
None
|
||||
}
|
||||
};
|
||||
if after.is_none() {
|
||||
put_cursor(index, &key, &format!("{version}:"))?;
|
||||
}
|
||||
let windows = cursors.get(&purge_key(room_id)).map(|value| parse_windows(value)).transpose()?.unwrap_or_default();
|
||||
let mut report = ChatReport::empty(room_id, facts.alias.clone());
|
||||
// The room's name is embedded and stored in every chunk, so it passes the scrub like a body.
|
||||
let label = match &facts.name {
|
||||
Some(name) => {
|
||||
let clean = scrub(name);
|
||||
for kind in clean.hits {
|
||||
report.scrubbed += 1;
|
||||
out.scrubbed.push(ScrubHit { room_id: room_id.to_owned(), event_id: "(name)".to_owned(), kind });
|
||||
}
|
||||
clean.text
|
||||
}
|
||||
None => facts.alias.clone().unwrap_or_else(|| room_id.to_owned()),
|
||||
};
|
||||
let room = Room { facts, label, windows };
|
||||
apply_purge(embedder, index, dimensions, &room, &mut report)?;
|
||||
let mut restarted = false;
|
||||
loop {
|
||||
let Some(page) = history.events_after(room_id, after.as_deref(), CHAT_PAGE)? else {
|
||||
ensure!(after.is_some() && !restarted, "the server does not know the start of {room_id}'s history");
|
||||
restarted = true;
|
||||
purge_room(index, room_id)?;
|
||||
put_cursor(index, &key, &format!("{version}:"))?;
|
||||
after = None;
|
||||
continue;
|
||||
};
|
||||
let Some(last) = page.events.last().map(|event| event.event_id.clone()) else { break };
|
||||
// A refusing event refuses the room before anything of the page is chunked or embedded.
|
||||
if let Some((event, reason)) = page.events.iter().find_map(|event| refuse_event(event).map(|reason| (event, reason))) {
|
||||
purge_room(index, room_id)?;
|
||||
return Ok(Outcome::Refused(RefusedRoom { room_id: room_id.to_owned(), reason, event_id: Some(event.event_id.clone()) }));
|
||||
}
|
||||
report.first.get_or_insert_with(|| page.events[0].event_id.clone());
|
||||
report.events += u32::try_from(page.events.len()).unwrap_or(u32::MAX);
|
||||
let folded = fold(&room, &page.events, &mut report, out);
|
||||
let Some(written) = write_page(embedder, index, dimensions, &room, folded, may_rebuild)? else {
|
||||
// Start this room over from the history; nothing it read is trusted to be complete.
|
||||
purge_room(index, room_id)?;
|
||||
put_cursor(index, &key, &format!("{version}:"))?;
|
||||
(after, may_rebuild) = (None, false);
|
||||
report = ChatReport { scrubbed: report.scrubbed, ..ChatReport::empty(room_id, facts.alias.clone()) };
|
||||
continue;
|
||||
};
|
||||
report.chunks += written;
|
||||
put_cursor(index, &key, &format!("{version}:{last}"))?;
|
||||
report.last = Some(last.clone());
|
||||
after = Some(last);
|
||||
if !page.more {
|
||||
break;
|
||||
}
|
||||
}
|
||||
report.seconds = started.elapsed().as_secs_f64();
|
||||
Ok(Outcome::Done(report))
|
||||
}
|
||||
|
||||
/// Brings the chat corpus to the reader's joined rooms. The collection must already exist and be
|
||||
/// Huginn's, built under this embedder's model: the forge refresh creates and rebuilds it, and this never does.
|
||||
/// A joined room the scope refuses is purged and named; a room the reader left is retired; the rest
|
||||
/// are ingested from their cursors. One room's failure is journaled and the others go on.
|
||||
/// `journal` receives every line the run prints: reports, scrub hits, refusals and failures.
|
||||
pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
history: &mut H,
|
||||
estate: &Estate,
|
||||
journal: &mut dyn FnMut(String),
|
||||
) -> Result<ChatRefreshed> {
|
||||
let DescribedSources::Labelled { meta, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||
bail!("the source collection is absent or not Huginn's; the forge refresh creates it and chat never does");
|
||||
};
|
||||
ensure!(meta.corpus == CORPUS, "the source collection was built for corpus {} and chat ingest writes {CORPUS}; the forge refresh rebuilds it", meta.corpus);
|
||||
embedder.built_under(Some(&meta.model_identity()));
|
||||
let identity = embedder.model_identity(BULK_EMBED_TIMEOUT).context("reading the embedding model's identity")?;
|
||||
ensure!(meta.model_identity() == identity, "the source collection was built under {} {} and chat embeds under {} {}", meta.model, meta.model_digest, identity.name, identity.digest);
|
||||
|
||||
let mut out = ChatRefreshed::default();
|
||||
let mut joined = history.joined_rooms().context("listing the reader's joined rooms")?;
|
||||
let mut seen = HashSet::new();
|
||||
joined.retain(|room| seen.insert(room.clone()));
|
||||
let mut keep: BTreeSet<String> = joined.iter().cloned().collect();
|
||||
let mut admitted: Vec<RoomFacts> = Vec::new();
|
||||
for room_id in &joined {
|
||||
let verdict = if estate.holds_id(room_id) {
|
||||
Ok(Err(Refusal::Estate))
|
||||
} else {
|
||||
history.facts(room_id).map(|facts| admit_room(&facts, estate).map(|()| facts))
|
||||
};
|
||||
match verdict {
|
||||
Ok(Ok(facts)) => admitted.push(facts),
|
||||
Ok(Err(reason)) => {
|
||||
// Not kept, so the retirement below purges whatever it held.
|
||||
keep.remove(room_id);
|
||||
journal(format!("refused: {room_id} {reason}"));
|
||||
out.refused.push(RefusedRoom { room_id: room_id.clone(), reason, event_id: None });
|
||||
}
|
||||
// An error is not a reclassification: the room keeps its points and cursor, and the report says why.
|
||||
Err(error) => {
|
||||
let report = ChatReport::failed(room_id, None, &error.context("reading the room's facts"));
|
||||
journal(report.to_string());
|
||||
out.failed.push(room_id.clone());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
}
|
||||
}
|
||||
match retire_chat(index, &keep) {
|
||||
Ok(done) => {
|
||||
for (room, why) in done.failed {
|
||||
journal(format!("huginn-sources: retiring {room} failed, the others go on: {why}"));
|
||||
out.failed.push(room);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
journal(format!("huginn-sources: retiring rooms failed: {error:#}"));
|
||||
out.failed.push("retire".to_owned());
|
||||
}
|
||||
}
|
||||
for facts in admitted {
|
||||
match ingest_room(embedder, index, history, identity.dimensions, &facts, &mut out) {
|
||||
Ok(Outcome::Done(report)) => {
|
||||
journal(report.to_string());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
Ok(Outcome::Refused(refused)) => {
|
||||
journal(format!("refused: {} {} [{}]", refused.room_id, refused.reason, refused.event_id.as_deref().unwrap_or("")));
|
||||
out.refused.push(refused);
|
||||
}
|
||||
Err(error) => {
|
||||
let report = ChatReport::failed(&facts.room_id, facts.alias.clone(), &error);
|
||||
journal(report.to_string());
|
||||
out.failed.push(facts.room_id.clone());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
}
|
||||
}
|
||||
for hit in &out.scrubbed {
|
||||
journal(format!("scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
166
crates/huginn-daemon/src/sources/chat_chunk.rs
Normal file
166
crates/huginn-daemon/src/sources/chat_chunk.rs
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
//! The chat chunker: greedy, deterministic and local. A chunk is consecutive
|
||||
//! messages of one room; it closes at a time gap, at a message count, or when
|
||||
//! its embedded text would pass a character bound. A message is never split.
|
||||
|
||||
use chrono::DateTime;
|
||||
use huginn_mind::redact::SCRUB_VERSION;
|
||||
use huginn_mind::ChatMessage;
|
||||
|
||||
use super::ChatPayload;
|
||||
use crate::index::point_id;
|
||||
|
||||
/// Bumped whenever chunk boundaries or the embedded text would change for the
|
||||
/// same messages. Part of `chat_version`.
|
||||
pub const CHAT_CHUNKER_VERSION: u32 = 1;
|
||||
|
||||
/// A chunk closes when the next message is more than this much later.
|
||||
pub const CHAT_GAP_MS: u64 = 30 * 60 * 1000;
|
||||
/// The most messages a chunk holds.
|
||||
pub const CHAT_MESSAGES_MAX: usize = 24;
|
||||
/// The most characters a chunk's embedded text holds.
|
||||
pub const CHAT_CHARS_MAX: usize = 1_600;
|
||||
/// The most characters a stored message body holds, marker included.
|
||||
pub const CHAT_BODY_CHARS_MAX: usize = 1_200;
|
||||
|
||||
/// Ends a body that was cut.
|
||||
const CUT_MARKER: &str = "…[cut]";
|
||||
|
||||
/// What the room's cursor and chunk ids are stamped with: the chunker and the
|
||||
/// scrub. Either changing re-ingests the room.
|
||||
pub fn chat_version() -> String {
|
||||
format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}")
|
||||
}
|
||||
|
||||
/// A chunk's id: where it starts and under which version it was cut.
|
||||
pub fn chunk_id(room_id: &str, first_event_id: &str) -> String {
|
||||
point_id(&format!("{room_id}\0{first_event_id}\0{}", chat_version()))
|
||||
}
|
||||
|
||||
/// A scrubbed body cut to `CHAT_BODY_CHARS_MAX` characters on a character boundary, with a marker.
|
||||
pub fn cut_body(body: &str) -> String {
|
||||
if body.chars().count() <= CHAT_BODY_CHARS_MAX {
|
||||
return body.to_owned();
|
||||
}
|
||||
let keep = CHAT_BODY_CHARS_MAX - CUT_MARKER.chars().count();
|
||||
body.chars().take(keep).chain(CUT_MARKER.chars()).collect()
|
||||
}
|
||||
|
||||
fn line_of(message: &ChatMessage) -> String {
|
||||
let when = DateTime::from_timestamp_millis(i64::try_from(message.ts_ms).unwrap_or(i64::MAX)).map_or_else(String::new, |at| at.format("%Y-%m-%d %H:%M").to_string());
|
||||
format!("{when} {}: {}", message.sender, message.body)
|
||||
}
|
||||
|
||||
/// What is embedded for a chunk: the room's label (its name, else its alias,
|
||||
/// else its id), then one `YYYY-MM-DD HH:MM sender: body` line per message, UTC.
|
||||
pub fn embedded_text(label: &str, messages: &[ChatMessage]) -> String {
|
||||
let mut text = label.to_owned();
|
||||
for message in messages {
|
||||
text.push('\n');
|
||||
text.push_str(&line_of(message));
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
/// `messages` (one room, in order) cut into chunks, each a run of consecutive messages.
|
||||
pub fn chunk_messages(label: &str, messages: &[ChatMessage]) -> Vec<Vec<ChatMessage>> {
|
||||
let mut chunks: Vec<Vec<ChatMessage>> = Vec::new();
|
||||
let mut chars = 0_usize;
|
||||
for message in messages {
|
||||
let line = line_of(message).chars().count() + 1;
|
||||
let closes = chunks.last().is_some_and(|open| {
|
||||
let last = open.last().map_or(0, |last| last.ts_ms);
|
||||
message.ts_ms.saturating_sub(last) > CHAT_GAP_MS || open.len() >= CHAT_MESSAGES_MAX || chars + line > CHAT_CHARS_MAX
|
||||
});
|
||||
if closes || chunks.is_empty() {
|
||||
chunks.push(Vec::new());
|
||||
chars = label.chars().count();
|
||||
}
|
||||
chunks.last_mut().expect("a chunk is open").push(message.clone());
|
||||
chars += line;
|
||||
}
|
||||
chunks
|
||||
}
|
||||
|
||||
/// The point payload of one chunk.
|
||||
pub fn chunk_payload(room_id: &str, alias: Option<&str>, label: &str, messages: Vec<ChatMessage>) -> ChatPayload {
|
||||
ChatPayload { room_id: room_id.to_owned(), room_alias: alias.map(str::to_owned), text: embedded_text(label, &messages), messages }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn message(n: u64, ts_ms: u64, body: &str) -> ChatMessage {
|
||||
ChatMessage { event_id: format!("$e{n}"), sender: "@a:s".into(), ts_ms, body: body.into() }
|
||||
}
|
||||
|
||||
fn ids(chunks: &[Vec<ChatMessage>]) -> Vec<Vec<&str>> {
|
||||
chunks.iter().map(|chunk| chunk.iter().map(|message| message.event_id.as_str()).collect()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gap_a_count_and_a_character_bound_each_close_a_chunk() {
|
||||
let gap = [message(1, 0, "a"), message(2, CHAT_GAP_MS, "b"), message(3, 2 * CHAT_GAP_MS + 1, "c")];
|
||||
assert_eq!(ids(&chunk_messages("r", &gap)), [["$e1", "$e2"].as_slice(), ["$e3"].as_slice()], "a gap of exactly the bound does not close");
|
||||
let many: Vec<ChatMessage> = (0..25).map(|n| message(n, n, "x")).collect();
|
||||
let chunks = chunk_messages("r", &many);
|
||||
assert_eq!((chunks[0].len(), chunks[1].len()), (CHAT_MESSAGES_MAX, 1));
|
||||
let long: Vec<ChatMessage> = (0..4).map(|n| message(n, n, &"w".repeat(500))).collect();
|
||||
let chunks = chunk_messages("r", &long);
|
||||
assert!(chunks.iter().all(|chunk| embedded_text("r", chunk).chars().count() <= CHAT_CHARS_MAX), "{:?}", ids(&chunks));
|
||||
assert!(chunks.len() > 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_embedded_text_is_the_label_then_utc_lines() {
|
||||
let text = embedded_text("Swarm", &[message(1, 1_700_000_000_000, "hello")]);
|
||||
assert_eq!(text, "Swarm\n2023-11-14 22:13 @a:s: hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_body_is_cut_at_the_bound_with_a_marker_on_a_char_boundary_and_a_short_one_is_whole() {
|
||||
let cut = cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX + 50));
|
||||
assert_eq!(cut.chars().count(), CHAT_BODY_CHARS_MAX);
|
||||
assert!(cut.ends_with(CUT_MARKER));
|
||||
assert_eq!(cut_body(&"é".repeat(CHAT_BODY_CHARS_MAX)), "é".repeat(CHAT_BODY_CHARS_MAX));
|
||||
let one = chunk_messages("r", &[message(1, 0, &cut)]);
|
||||
assert_eq!(one.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chunk_id_is_its_room_first_event_and_version() {
|
||||
assert_eq!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$a"));
|
||||
assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b"));
|
||||
assert_ne!(chunk_id("!r:s", "$a"), chunk_id("!q:s", "$a"));
|
||||
assert_eq!(chat_version(), format!("{CHAT_CHUNKER_VERSION}.{SCRUB_VERSION}"));
|
||||
}
|
||||
|
||||
/// Chunking the whole sequence equals re-chunking the open tail with each new batch appended.
|
||||
#[test]
|
||||
fn appending_to_the_open_tail_gives_the_boundaries_of_chunking_all_at_once() {
|
||||
let mut seed = 0x2545_f491_4f6c_dd1d_u64;
|
||||
let mut next = move |bound: u64| {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 7;
|
||||
seed ^= seed << 17;
|
||||
seed % bound
|
||||
};
|
||||
let mut ts = 0;
|
||||
let all: Vec<ChatMessage> = (0..300)
|
||||
.map(|n| {
|
||||
ts += if next(7) == 0 { CHAT_GAP_MS + 1 } else { next(60_000) };
|
||||
message(n, ts, &"z".repeat(next(700) as usize + 1))
|
||||
})
|
||||
.collect();
|
||||
let whole = chunk_messages("r", &all);
|
||||
for step in [1_usize, 5, 17, 100] {
|
||||
let mut held: Vec<Vec<ChatMessage>> = Vec::new();
|
||||
for batch in all.chunks(step) {
|
||||
let mut tail: Vec<ChatMessage> = held.pop().unwrap_or_default();
|
||||
tail.extend_from_slice(batch);
|
||||
held.extend(chunk_messages("r", &tail));
|
||||
}
|
||||
assert_eq!(held, whole, "step {step}");
|
||||
}
|
||||
}
|
||||
}
|
||||
103
crates/huginn-daemon/src/sources/chat_purge.rs
Normal file
103
crates/huginn-daemon/src/sources/chat_purge.rs
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
//! A steward's purge of a room's time window. The tombstone is the one chat
|
||||
//! state that is not derived from Synapse: a durable request, kept in the
|
||||
//! collection's cursors object under `chat-purge:<room_id>`, that must outlive
|
||||
//! the points it removes, so a re-ingest never brings the window back.
|
||||
//!
|
||||
//! `request_purge` is the only writer; `refresh_chat` reads the tombstone and
|
||||
//! applies it at its next run. The code does not authenticate the steward: whoever
|
||||
//! can run the job as the unit's user (the host admin) may ask, and the request
|
||||
//! carries a ref in the job's journal for a trace. Tombstones live in the
|
||||
//! collection, so a full collection rebuild forgets them; re-issue them from
|
||||
//! their refs.
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
|
||||
use super::chat::CHAT_PURGE_PREFIX;
|
||||
use super::{DescribedSources, SourceIndex};
|
||||
use crate::index::INDEX_CALL_TIMEOUT;
|
||||
|
||||
/// The most windows a room's tombstone holds.
|
||||
pub const PURGE_WINDOWS_MAX: usize = 32;
|
||||
|
||||
/// Messages with `from_ms <= ts_ms < to_ms` are purged.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct PurgeWindow {
|
||||
pub from_ms: u64,
|
||||
pub to_ms: u64,
|
||||
}
|
||||
|
||||
impl PurgeWindow {
|
||||
pub fn covers(&self, ts_ms: u64) -> bool {
|
||||
self.from_ms <= ts_ms && ts_ms < self.to_ms
|
||||
}
|
||||
}
|
||||
|
||||
/// The tombstone key of a room.
|
||||
pub fn purge_key(room_id: &str) -> String {
|
||||
format!("{CHAT_PURGE_PREFIX}{room_id}")
|
||||
}
|
||||
|
||||
/// A tombstone value, `<from>..<to>[,<from>..<to>...]`.
|
||||
pub fn parse_windows(value: &str) -> Result<Vec<PurgeWindow>> {
|
||||
value
|
||||
.split(',')
|
||||
.map(|part| {
|
||||
let (from, to) = part.split_once("..").with_context(|| format!("purge window {part:?} is not <from>..<to>"))?;
|
||||
let window = PurgeWindow { from_ms: from.parse().context("a purge window's start is not milliseconds")?, to_ms: to.parse().context("a purge window's end is not milliseconds")? };
|
||||
if window.from_ms >= window.to_ms {
|
||||
bail!("purge window {part:?} is empty");
|
||||
}
|
||||
Ok(window)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Windows sorted, with overlapping and touching ones joined.
|
||||
pub fn merge(mut windows: Vec<PurgeWindow>) -> Vec<PurgeWindow> {
|
||||
windows.sort();
|
||||
let mut merged: Vec<PurgeWindow> = Vec::new();
|
||||
for window in windows {
|
||||
match merged.last_mut() {
|
||||
Some(last) if window.from_ms <= last.to_ms => last.to_ms = last.to_ms.max(window.to_ms),
|
||||
_ => merged.push(window),
|
||||
}
|
||||
}
|
||||
merged
|
||||
}
|
||||
|
||||
pub fn encode(windows: &[PurgeWindow]) -> String {
|
||||
windows.iter().map(|window| format!("{}..{}", window.from_ms, window.to_ms)).collect::<Vec<_>>().join(",")
|
||||
}
|
||||
|
||||
/// Whether any window covers `ts_ms`.
|
||||
pub fn covers(windows: &[PurgeWindow], ts_ms: u64) -> bool {
|
||||
windows.iter().any(|window| window.covers(ts_ms))
|
||||
}
|
||||
|
||||
/// Records that `window` of `room_id` must leave the index and never return. Writes the cursors
|
||||
/// object only if the tombstone changed, and refuses a room's 33rd window. Writes no point and
|
||||
/// embeds nothing; the next `refresh_chat` applies it. The job's `chat-purge` subcommand calls
|
||||
/// it under the job's lock, because `set_cursors` replaces the cursors object whole.
|
||||
/// Returns whether the tombstone changed.
|
||||
pub fn request_purge<S: SourceIndex>(index: &mut S, room_id: &str, window: PurgeWindow) -> Result<bool> {
|
||||
if window.from_ms >= window.to_ms {
|
||||
bail!("a purge window must start before it ends");
|
||||
}
|
||||
let DescribedSources::Labelled { mut cursors, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||
bail!("the source collection is absent or not Huginn's; the forge refresh creates it");
|
||||
};
|
||||
let key = purge_key(room_id);
|
||||
let mut windows = cursors.get(&key).map(|held| parse_windows(held)).transpose()?.unwrap_or_default();
|
||||
windows.push(window);
|
||||
let windows = merge(windows);
|
||||
if windows.len() > PURGE_WINDOWS_MAX {
|
||||
bail!("room {room_id} already holds {PURGE_WINDOWS_MAX} purge windows; widen one instead");
|
||||
}
|
||||
let encoded = encode(&windows);
|
||||
if cursors.get(&key) == Some(&encoded) {
|
||||
return Ok(false);
|
||||
}
|
||||
cursors.insert(key, encoded);
|
||||
index.set_cursors(&cursors).context("writing the purge tombstone")?;
|
||||
Ok(true)
|
||||
}
|
||||
964
crates/huginn-daemon/src/sources/chat_tests.rs
Normal file
964
crates/huginn-daemon/src/sources/chat_tests.rs
Normal file
|
|
@ -0,0 +1,964 @@
|
|||
//! Chat ingest against a scripted room history, a fake vector store and the fake embedder.
|
||||
//! The refusals come first: what must never be read, embedded or stored.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
use huginn_mind::ChatMessage;
|
||||
use huginn_mind::redact::ScrubKind;
|
||||
|
||||
use super::chat::*;
|
||||
use super::chat_chunk::{CHAT_BODY_CHARS_MAX, chat_version, chunk_id};
|
||||
use super::chat_purge::{PURGE_WINDOWS_MAX, PurgeWindow, request_purge};
|
||||
use super::room_scope::{ESTATE_ALIASES, Estate, Refusal, RoomFacts};
|
||||
use super::*;
|
||||
use crate::index::MANAGED_BY;
|
||||
use crate::index::fakes::{FakeEmbedder, FakeSources};
|
||||
|
||||
// ---- the rig ----
|
||||
|
||||
fn plain(id: &str) -> RoomFacts {
|
||||
RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None }
|
||||
}
|
||||
|
||||
fn estate(ids: &[&str]) -> Estate {
|
||||
Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap()
|
||||
}
|
||||
|
||||
struct Scripted {
|
||||
facts: RoomFacts,
|
||||
events: Vec<RoomEvent>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FakeHistory {
|
||||
joined: Vec<String>,
|
||||
rooms: BTreeMap<String, Scripted>,
|
||||
calls: Vec<String>,
|
||||
/// Rooms whose state cannot be read.
|
||||
facts_fail: BTreeSet<String>,
|
||||
/// Serve an event a redaction targets as the server does, without its content.
|
||||
prune: bool,
|
||||
}
|
||||
|
||||
impl FakeHistory {
|
||||
fn room(&mut self, id: &str, events: Vec<RoomEvent>) -> &mut Self {
|
||||
self.rooms.insert(id.into(), Scripted { facts: plain(id), events });
|
||||
self.joined.push(id.into());
|
||||
self
|
||||
}
|
||||
|
||||
fn facts(&mut self, id: &str, edit: impl FnOnce(&mut RoomFacts)) -> &mut Self {
|
||||
edit(&mut self.rooms.get_mut(id).unwrap().facts);
|
||||
self
|
||||
}
|
||||
|
||||
fn push(&mut self, id: &str, events: Vec<RoomEvent>) {
|
||||
self.rooms.get_mut(id).unwrap().events.extend(events);
|
||||
}
|
||||
|
||||
fn called(&self, prefix: &str, room: &str) -> bool {
|
||||
self.calls.iter().any(|call| call == &format!("{prefix}:{room}"))
|
||||
}
|
||||
}
|
||||
|
||||
impl RoomHistory for FakeHistory {
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||
self.calls.push("joined".into());
|
||||
Ok(self.joined.clone())
|
||||
}
|
||||
|
||||
fn facts(&mut self, room_id: &str) -> Result<RoomFacts> {
|
||||
self.calls.push(format!("facts:{room_id}"));
|
||||
if self.facts_fail.contains(room_id) {
|
||||
bail!("the server did not answer for the state of {room_id}");
|
||||
}
|
||||
match self.rooms.get(room_id) {
|
||||
Some(room) => Ok(room.facts.clone()),
|
||||
None => bail!("no such room"),
|
||||
}
|
||||
}
|
||||
|
||||
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>> {
|
||||
self.calls.push(format!("events:{room_id}"));
|
||||
let Some(room) = self.rooms.get(room_id) else { bail!("no such room") };
|
||||
let from = match after {
|
||||
None => 0,
|
||||
Some(after) => match room.events.iter().position(|event| event.event_id == after) {
|
||||
Some(at) => at + 1,
|
||||
None => return Ok(None),
|
||||
},
|
||||
};
|
||||
let redacted: BTreeSet<&str> = room.events.iter().filter_map(|event| if let EventKind::Redaction { target } = &event.kind { Some(target.as_str()) } else { None }).collect();
|
||||
let served: Vec<RoomEvent> = room.events[from..]
|
||||
.iter()
|
||||
.map(|event| if self.prune && redacted.contains(event.event_id.as_str()) { RoomEvent { kind: EventKind::Other, content_keys: vec![], ..event.clone() } } else { event.clone() })
|
||||
.collect();
|
||||
let take = served.len().min(limit as usize);
|
||||
Ok(Some(EventPage { events: served[..take].to_vec(), more: take < served.len() }))
|
||||
}
|
||||
}
|
||||
|
||||
fn event(id: &str, sender: &str, ts_ms: u64, kind: EventKind) -> RoomEvent {
|
||||
RoomEvent { event_id: id.into(), sender: sender.into(), ts_ms, content_keys: vec!["body".into(), "msgtype".into()], kind }
|
||||
}
|
||||
|
||||
fn said(id: &str, ts_ms: u64, body: &str) -> RoomEvent {
|
||||
event(id, "@a:s", ts_ms, EventKind::Message { body: body.into() })
|
||||
}
|
||||
|
||||
fn edit(id: &str, sender: &str, ts_ms: u64, target: &str, body: &str) -> RoomEvent {
|
||||
event(id, sender, ts_ms, EventKind::Edit { target: target.into(), body: body.into() })
|
||||
}
|
||||
|
||||
fn redaction(id: &str, ts_ms: u64, target: &str) -> RoomEvent {
|
||||
event(id, "@a:s", ts_ms, EventKind::Redaction { target: target.into() })
|
||||
}
|
||||
|
||||
struct Rig {
|
||||
embedder: FakeEmbedder,
|
||||
index: FakeSources,
|
||||
estate: Estate,
|
||||
}
|
||||
|
||||
impl Rig {
|
||||
fn new() -> Self {
|
||||
let mut embedder = FakeEmbedder::new("d1");
|
||||
let index = FakeSources::default();
|
||||
let identity = crate::index::Embedder::model_identity(&mut embedder, Duration::from_secs(1)).unwrap();
|
||||
let meta = SourcesMeta {
|
||||
managed_by: MANAGED_BY.into(),
|
||||
corpus: CORPUS.into(),
|
||||
model: identity.name,
|
||||
model_digest: identity.digest,
|
||||
dimensions: identity.dimensions,
|
||||
chunker_version: chunk::CHUNKER_VERSION,
|
||||
};
|
||||
index.plant(DescribedSources::Labelled { meta, cursors: BTreeMap::new() });
|
||||
Self { embedder, index, estate: estate(&[]) }
|
||||
}
|
||||
|
||||
fn run(&mut self, history: &mut FakeHistory) -> (Result<ChatRefreshed>, Vec<String>) {
|
||||
let mut journal = Vec::new();
|
||||
let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line));
|
||||
(out, journal)
|
||||
}
|
||||
|
||||
fn ok(&mut self, history: &mut FakeHistory) -> ChatRefreshed {
|
||||
self.run(history).0.unwrap()
|
||||
}
|
||||
|
||||
fn points(&self) -> Vec<ChatPoint> {
|
||||
self.index.state.lock().unwrap().chat.values().cloned().collect()
|
||||
}
|
||||
|
||||
fn ids(&self) -> BTreeSet<String> {
|
||||
self.index.state.lock().unwrap().chat.keys().cloned().collect()
|
||||
}
|
||||
|
||||
fn bodies(&self) -> Vec<String> {
|
||||
let mut bodies: Vec<String> = self.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| message.body.clone())).collect();
|
||||
bodies.sort();
|
||||
bodies
|
||||
}
|
||||
|
||||
fn held_ids(&self, room: &str) -> Vec<String> {
|
||||
let mut ids: Vec<String> =
|
||||
self.points().iter().filter(|point| point.payload.room_id == room).flat_map(|point| point.payload.messages.iter().map(|message| message.event_id.clone())).collect();
|
||||
ids.sort();
|
||||
ids
|
||||
}
|
||||
|
||||
fn cursor(&self, room: &str) -> Option<String> {
|
||||
self.index.cursors().get(&cursor_key(room)).cloned()
|
||||
}
|
||||
|
||||
fn embedded(&self) -> Vec<String> {
|
||||
self.embedder.texts()
|
||||
}
|
||||
|
||||
fn seed(&mut self, room: &str, event_id: &str, ts: u64) {
|
||||
let mut seeded = vec![ChatPoint {
|
||||
id: chunk_id(room, event_id),
|
||||
vector: vec![1.0; 4],
|
||||
payload: chunk_payload_of(room, &[(event_id, ts)]),
|
||||
}];
|
||||
self.index.upsert_chat(&seeded.split_off(0)).unwrap();
|
||||
let mut cursors = self.index.cursors();
|
||||
cursors.insert(cursor_key(room), format!("{}:{event_id}", chat_version()));
|
||||
self.index.set_cursors(&cursors).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn chunk_payload_of(room: &str, messages: &[(&str, u64)]) -> ChatPayload {
|
||||
let messages: Vec<ChatMessage> = messages.iter().map(|(id, ts)| ChatMessage { event_id: (*id).into(), sender: "@a:s".into(), ts_ms: *ts, body: format!("seeded {id}") }).collect();
|
||||
ChatPayload { room_id: room.into(), room_alias: None, text: "seeded".into(), messages }
|
||||
}
|
||||
|
||||
fn tombstone(rig: &Rig, room: &str) -> Option<String> {
|
||||
rig.index.cursors().get(&format!("{CHAT_PURGE_PREFIX}{room}")).cloned()
|
||||
}
|
||||
|
||||
// ---- scope: refusals first ----
|
||||
|
||||
#[test]
|
||||
fn only_joined_rooms_are_looked_at_and_a_join_rule_is_not_the_test() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!a:s", vec![said("$a1", 1, "in a")]).room("!b:s", vec![said("$b1", 2, "in b")]);
|
||||
// A room the reader has not joined: scripted, but never listed.
|
||||
history.rooms.insert("!c:s".into(), Scripted { facts: plain("!c:s"), events: vec![said("$c1", 3, "in c")] });
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.is_clean());
|
||||
assert!(!history.calls.iter().any(|call| call.contains("!c:s")), "{:?}", history.calls);
|
||||
assert_eq!(rig.held_ids("!a:s"), ["$a1"]);
|
||||
assert_eq!(rig.held_ids("!b:s"), ["$b1"]);
|
||||
assert!(rig.held_ids("!c:s").is_empty() && done.rooms.iter().all(|report| report.room_id != "!c:s"));
|
||||
let code = include_str!("room_scope.rs");
|
||||
assert!(!code[..code.find("#[cfg(test)]").unwrap()].contains(&["join", "_rule"].concat()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dm_room_is_refused_unread_unindexed_and_purged() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!dm:s", vec![said("$d1", 1, "private")]).facts("!dm:s", |facts| facts.direct = true);
|
||||
let mut rig = Rig::new();
|
||||
rig.seed("!dm:s", "$old", 5);
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(!history.called("events", "!dm:s"));
|
||||
assert!(rig.points().is_empty() && rig.cursor("!dm:s").is_none() && rig.embedded().is_empty());
|
||||
assert_eq!(done.refused, [RefusedRoom { room_id: "!dm:s".into(), reason: Refusal::Direct, event_id: None }]);
|
||||
assert!(!done.is_clean() && journal.iter().any(|line| line == "refused: !dm:s direct room"), "{journal:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_estate_room_is_refused_by_id_and_by_alias_and_an_excluded_room_in_the_index_is_purged() {
|
||||
let mut rig = Rig::new();
|
||||
rig.estate = estate(&["!byid:s"]);
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!byid:s", vec![said("$i1", 1, "x")]);
|
||||
for (n, alias) in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:GAMECULT.ORG".to_owned()]).enumerate() {
|
||||
history.room(&format!("!al{n}:s"), vec![said("$x", 1, "x")]).facts(&format!("!al{n}:s"), |facts| facts.alias = Some(alias));
|
||||
}
|
||||
history.room("!alt:s", vec![said("$x", 1, "x")]).facts("!alt:s", |facts| facts.alt_aliases = vec!["#other:s".into(), "#patrons:gamecult.org".into()]);
|
||||
history.room("!ok:s", vec![said("$ok1", 1, "fine")]);
|
||||
history.room("!guide:s", vec![said("$g1", 1, "fine")]).facts("!guide:s", |facts| facts.alias = Some("#users-guide:gamecult.org".into()));
|
||||
history.room("!club:s", vec![said("$c1", 1, "fine")]).facts("!club:s", |facts| facts.alias = Some("#patrons-club:gamecult.org".into()));
|
||||
for room in ["!byid:s", "!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] {
|
||||
rig.seed(room, "$old", 5);
|
||||
}
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert(format!("{CHAT_PURGE_PREFIX}!al0:s"), "1..2".into());
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(!history.called("facts", "!byid:s") && !history.called("events", "!byid:s"));
|
||||
for room in ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s"] {
|
||||
assert!(history.called("facts", room) && !history.called("events", room), "{room}");
|
||||
}
|
||||
let refused: BTreeSet<&str> = done.refused.iter().map(|refused| refused.room_id.as_str()).collect();
|
||||
assert_eq!(refused, ["!al0:s", "!al1:s", "!al2:s", "!al3:s", "!alt:s", "!byid:s"].into());
|
||||
assert!(done.refused.iter().all(|refused| refused.reason == Refusal::Estate) && !done.is_clean());
|
||||
let held: BTreeSet<String> = rig.points().iter().map(|point| point.payload.room_id.clone()).collect();
|
||||
assert_eq!(held, ["!guide:s".to_owned(), "!club:s".to_owned(), "!ok:s".to_owned()].into(), "lookalikes and plain rooms are in; every estate room is out");
|
||||
assert!(rig.index.cursors().keys().all(|key| !key.starts_with("chat:!al") && !key.starts_with("chat:!byid") && !key.starts_with("chat:!alt")));
|
||||
assert_eq!(tombstone(&rig, "!al0:s").as_deref(), Some("1..2"), "a tombstone is never touched by a refusal");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_encrypted_room_is_refused_and_an_encrypted_event_in_a_plain_room_is_never_stored() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!enc:s", vec![said("$e1", 1, "x")]).facts("!enc:s", |facts| facts.encrypted = true);
|
||||
history.room("!plain:s", vec![said("$p1", 1, "visible"), event("$p2", "@a:s", 2, EventKind::Other), said("$p3", 3, "also visible")]);
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert_eq!(done.refused[0].reason, Refusal::Encrypted);
|
||||
assert!(!history.called("events", "!enc:s"));
|
||||
assert_eq!(rig.held_ids("!plain:s"), ["$p1", "$p3"]);
|
||||
assert_eq!(rig.cursor("!plain:s").unwrap(), format!("{}:$p3", chat_version()), "the cursor passes an Other event");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bridged_room_is_refused_whole_before_any_event_is_read_and_its_chunks_are_purged() {
|
||||
let mut history = FakeHistory::default();
|
||||
for (n, state_type) in ["m.bridge", "uk.half-shot.bridge"].into_iter().enumerate() {
|
||||
let room = format!("!br{n}:s");
|
||||
history.room(&room, vec![said("$x", 1, "x")]).facts(&room, |facts| facts.bridge = Some(state_type.into()));
|
||||
}
|
||||
let mut rig = Rig::new();
|
||||
rig.seed("!br0:s", "$old", 5);
|
||||
let done = rig.ok(&mut history);
|
||||
assert_eq!(done.refused.len(), 2);
|
||||
assert!(done.refused.iter().all(|refused| matches!(refused.reason, Refusal::Bridged(_))));
|
||||
assert!(!history.calls.iter().any(|call| call.starts_with("events:")));
|
||||
assert!(rig.points().is_empty() && rig.index.cursors().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bridged_platforms_sender_or_content_refuses_the_room_before_anything_is_embedded_and_purges_what_it_held() {
|
||||
let cases = [
|
||||
event("$bad", "@discord_123:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@_discord_9:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@mautrix_discord:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
event("$bad", "@_bridge_discordbot:s", 2, EventKind::Message { body: "SECRET-BODY-TEXT".into() }),
|
||||
RoomEvent { content_keys: vec!["body".into(), "fi.mau.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") },
|
||||
RoomEvent { content_keys: vec!["uk.half-shot.discord".into()], ..said("$bad", 2, "SECRET-BODY-TEXT") },
|
||||
];
|
||||
for bad in cases {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$ok", 1, "before"), bad]);
|
||||
history.room("!other:s", vec![said("$o1", 1, "other room")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.seed("!r:s", "$prev", 0);
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(!rig.embedded().iter().any(|text| text.contains("SECRET-BODY-TEXT") || text.contains("before")), "nothing of the page is embedded");
|
||||
assert!(rig.held_ids("!r:s").is_empty() && rig.cursor("!r:s").is_none());
|
||||
assert_eq!(rig.held_ids("!other:s"), ["$o1"]);
|
||||
assert_eq!(done.refused.len(), 1);
|
||||
assert_eq!((done.refused[0].room_id.as_str(), done.refused[0].event_id.as_deref()), ("!r:s", Some("$bad")));
|
||||
assert!(!done.is_clean());
|
||||
assert!(journal.iter().all(|line| !line.contains("SECRET-BODY-TEXT")), "{journal:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lookalike_of_the_bridged_platform_is_kept() {
|
||||
let mut history = FakeHistory::default();
|
||||
let kept = ["@discourse:s", "@disco:s", "@eureka:s"].map(|sender| event(&format!("${}", &sender[1..6]), sender, 1, EventKind::Message { body: "hi".into() }));
|
||||
let mut keyed = said("$k", 2, "hi");
|
||||
keyed.content_keys.push("discourse".into());
|
||||
let mut events = kept.to_vec();
|
||||
events.push(keyed);
|
||||
history.room("!r:s", events);
|
||||
let mut rig = Rig::new();
|
||||
assert!(rig.ok(&mut history).is_clean());
|
||||
assert_eq!(rig.held_ids("!r:s").len(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_collection_is_never_created_by_chat() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||
for described in [None, Some(DescribedSources::Absent), Some(DescribedSources::Unlabelled)] {
|
||||
let mut rig = Rig::new();
|
||||
match described {
|
||||
None => rig.index.state.lock().unwrap().described = None,
|
||||
Some(described) => rig.index.plant(described),
|
||||
}
|
||||
let error = format!("{:#}", rig.run(&mut history).0.unwrap_err());
|
||||
assert!(error.contains("forge refresh creates it"), "{error}");
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, 0);
|
||||
assert!(rig.points().is_empty() && rig.embedded().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
// ---- redactions, edits, replays ----
|
||||
|
||||
#[test]
|
||||
fn a_redacted_event_is_absent_after_the_next_refresh_and_a_first_or_only_message_changes_or_drops_its_chunk() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), said("$c", 30, "gamma")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let first_id = chunk_id("!r:s", "$a");
|
||||
assert_eq!(rig.ids(), [first_id.clone()].into());
|
||||
history.push("!r:s", vec![redaction("$r1", 40, "$b")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$c"]);
|
||||
assert!(!rig.points()[0].payload.text.contains("beta"));
|
||||
history.push("!r:s", vec![redaction("$r2", 50, "$a")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids(), [chunk_id("!r:s", "$c")].into(), "the first message left, so the id changed and the old one is gone");
|
||||
history.push("!r:s", vec![redaction("$r3", 60, "$c")]);
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.points().is_empty(), "a chunk the redaction empties is deleted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redaction_in_the_same_page_as_its_message_never_reaches_the_embedder() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "keep me"), said("$gone", 20, "REDACTED-BODY"), redaction("$r", 30, "$gone")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("REDACTED-BODY")));
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_edit_replaces_the_body_in_place_and_a_stranger_an_unknown_target_or_an_edit_of_an_edit_change_nothing() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "old words"), said("$b", 20, "other")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let before = rig.points();
|
||||
history.push("!r:s", vec![edit("$e1", "@a:s", 30, "$a", "new words")]);
|
||||
rig.ok(&mut history);
|
||||
let after = rig.points();
|
||||
assert_eq!(after.len(), 1);
|
||||
let first = &after[0].payload.messages[0];
|
||||
assert_eq!((first.event_id.as_str(), first.sender.as_str(), first.ts_ms, first.body.as_str()), ("$a", "@a:s", 10, "new words"));
|
||||
assert_eq!(after[0].id, before[0].id);
|
||||
assert!(after[0].payload.text.contains("new words") && !after[0].payload.text.contains("old words"));
|
||||
assert!(rig.embedded().last().unwrap().contains("new words"));
|
||||
let calls = rig.embedder.state.lock().unwrap().embedded.len();
|
||||
history.push("!r:s", vec![edit("$e2", "@stranger:s", 40, "$a", "hijack"), edit("$e3", "@a:s", 41, "$nothing", "lost"), edit("$e4", "@a:s", 42, "$e1", "edit of an edit")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.points(), after);
|
||||
assert_eq!(rig.embedder.state.lock().unwrap().embedded.len(), calls, "no embed call for edits that change nothing");
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$e4", chat_version()), "the cursor is the last event of any kind");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_after_a_crash_between_the_upsert_and_the_cursor_neither_duplicates_nor_loses() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta"), edit("$e", "@a:s", 25, "$a", "alpha two"), said("$c", 30, "gamma")]);
|
||||
let mut rig = Rig::new();
|
||||
// One cursor write (the empty start) succeeds, then the page's upsert is acknowledged and the next write fails.
|
||||
rig.index.state.lock().unwrap().fail_cursor_writes_after = Some(1);
|
||||
assert!(!rig.run(&mut history).0.unwrap().failed.is_empty());
|
||||
let (ids, held) = (rig.ids(), rig.held_ids("!r:s"));
|
||||
assert_eq!(held, ["$a", "$b", "$c"]);
|
||||
rig.index.state.lock().unwrap().fail_cursor_writes_after = None;
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.is_clean());
|
||||
assert_eq!((rig.ids(), rig.held_ids("!r:s")), (ids, held));
|
||||
assert!(rig.bodies().contains(&"alpha two".to_owned()));
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$c", chat_version()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_upsert_leaves_the_cursor_and_a_failed_embed_or_delete_deletes_nothing_the_index_held() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let (cursor, points) = (rig.cursor("!r:s"), rig.points());
|
||||
history.push("!r:s", vec![said("$c", 30, "gamma")]);
|
||||
rig.index.state.lock().unwrap().fail_chat_upserts = true;
|
||||
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone()));
|
||||
rig.index.state.lock().unwrap().fail_chat_upserts = false;
|
||||
rig.embedder.state.lock().unwrap().embed_fails = true;
|
||||
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor.clone(), points.clone()));
|
||||
rig.embedder.state.lock().unwrap().embed_fails = false;
|
||||
// A redaction of the first message writes a new chunk, then fails to delete the old one: the cursor stays.
|
||||
history.push("!r:s", vec![redaction("$r", 40, "$a")]);
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes = true;
|
||||
assert_eq!(rig.run(&mut history).0.unwrap().failed, ["!r:s"]);
|
||||
assert_eq!(rig.cursor("!r:s"), cursor);
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes = false;
|
||||
assert!(rig.ok(&mut history).is_clean());
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$b", "$c"]);
|
||||
}
|
||||
|
||||
// ---- chunking through the job ----
|
||||
|
||||
#[test]
|
||||
fn appending_to_the_tail_over_many_runs_equals_one_run_over_all() {
|
||||
let all: Vec<RoomEvent> = (0..80).map(|n| said(&format!("$m{n:03}"), 1_000 * n + if n >= 40 { 5_000_000 } else { 0 }, &format!("message number {n} {}", "x".repeat((n as usize * 37) % 400)))).collect();
|
||||
let mut whole_history = FakeHistory::default();
|
||||
whole_history.room("!r:s", all.clone());
|
||||
let mut whole = Rig::new();
|
||||
whole.ok(&mut whole_history);
|
||||
let mut grown_history = FakeHistory::default();
|
||||
grown_history.room("!r:s", vec![]);
|
||||
let mut grown = Rig::new();
|
||||
for batch in all.chunks(7) {
|
||||
grown_history.push("!r:s", batch.to_vec());
|
||||
grown.ok(&mut grown_history);
|
||||
}
|
||||
assert!(whole.points().len() > 2);
|
||||
let shape = |rig: &Rig| rig.points().into_iter().map(|point| (point.id, point.payload)).collect::<Vec<_>>();
|
||||
assert_eq!(shape(&grown), shape(&whole));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_message_is_cut_with_a_marker_and_every_point_satisfies_the_payload_contract() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, &"é".repeat(CHAT_BODY_CHARS_MAX * 2)), said("$b", 20, "short"), said("$c", 99_000_000, "later")]);
|
||||
history.facts("!r:s", |facts| {
|
||||
facts.alias = Some("#swarm:gamecult.org".into());
|
||||
facts.name = Some("Swarm".into());
|
||||
});
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let points = rig.points();
|
||||
assert_eq!(points.len(), 2);
|
||||
for point in &points {
|
||||
let payload = &point.payload;
|
||||
assert_eq!((payload.room_id.as_str(), payload.room_alias.as_deref()), ("!r:s", Some("#swarm:gamecult.org")));
|
||||
assert_eq!(point.id, chunk_id("!r:s", &payload.messages[0].event_id));
|
||||
assert_eq!(payload.text, super::chat_chunk::embedded_text("Swarm", &payload.messages));
|
||||
assert!(!payload.messages.is_empty());
|
||||
}
|
||||
let long = &points.iter().find(|point| point.payload.messages[0].event_id == "$a").unwrap().payload.messages[0];
|
||||
assert_eq!(long.body.chars().count(), CHAT_BODY_CHARS_MAX);
|
||||
assert!(long.body.ends_with("[cut]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lost_cursor_event_or_a_version_bump_re_ingests_that_room_alone() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 10, "alpha"), said("$b", 20, "beta")]).room("!q:s", vec![said("$q1", 10, "quiet")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let q_ids: BTreeSet<String> = rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect();
|
||||
let recreated = rig.index.state.lock().unwrap().recreated;
|
||||
for bump in [Some("0.0".to_owned()), None] {
|
||||
let mut cursors = rig.index.cursors();
|
||||
let value = match &bump {
|
||||
Some(old) => format!("{old}:$b"),
|
||||
None => format!("{}:$gone-from-the-server", chat_version()),
|
||||
};
|
||||
cursors.insert(cursor_key("!r:s"), value);
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
let before = rig.embedded().len();
|
||||
assert!(rig.ok(&mut history).is_clean());
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$b"]);
|
||||
assert!(rig.embedded().len() > before, "the room was re-embedded");
|
||||
assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!q:s").map(|point| point.id.clone()).collect::<BTreeSet<_>>(), q_ids);
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$b", chat_version()));
|
||||
}
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, recreated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retire_chat_removes_rooms_no_longer_kept_and_given_none_all_chat_and_touches_nothing_else() {
|
||||
let mut rig = Rig::new();
|
||||
rig.index.upsert_sources(&[SourcePoint {
|
||||
id: "forge-1".into(),
|
||||
vector: vec![1.0; 4],
|
||||
payload: SourcePayload { repo: "Repo".into(), path: "a".into(), commit: "c".into(), blob: "b".into(), line_start: 1, line_end: 1, chunk_index: 0, ext: "rs".into(), text: "t".into() },
|
||||
}])
|
||||
.unwrap();
|
||||
for room in ["!a:s", "!b:s"] {
|
||||
rig.seed(room, "$e", 1);
|
||||
}
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert("Repo".into(), "c".into());
|
||||
cursors.insert(format!("{CHAT_PURGE_PREFIX}!a:s"), "1..2".into());
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
let retired = retire_chat(&mut rig.index, &["!a:s".to_owned()].into()).unwrap();
|
||||
assert_eq!(retired.retired, ["!b:s"]);
|
||||
assert_eq!(rig.held_ids("!a:s"), ["$e"]);
|
||||
assert!(rig.held_ids("!b:s").is_empty() && rig.cursor("!b:s").is_none());
|
||||
assert_eq!(retire_chat(&mut rig.index, &BTreeSet::new()).unwrap().retired, ["!a:s"]);
|
||||
assert!(rig.points().is_empty());
|
||||
assert_eq!(rig.index.state.lock().unwrap().points.len(), 1, "a forge point stays");
|
||||
let kept: Vec<String> = rig.index.cursors().into_keys().collect();
|
||||
assert_eq!(kept, ["Repo".to_owned(), format!("{CHAT_PURGE_PREFIX}!a:s")]);
|
||||
// A room the reader left is retired by the refresh itself.
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||
rig.seed("!gone:s", "$e", 1);
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.held_ids("!gone:s").is_empty());
|
||||
}
|
||||
|
||||
// ---- the credential scrub through the job ----
|
||||
|
||||
fn planted() -> Vec<(ScrubKind, String, String)> {
|
||||
let t = |kind, secret: String, needle: &str| (kind, secret, needle.to_owned());
|
||||
vec![
|
||||
t(ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2a1B2"),
|
||||
t(ScrubKind::Bearer, format!("Bearer {}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"),
|
||||
t(ScrubKind::Bearer, format!("bearer:{}", "z9Y8".repeat(6)), "z9Y8z9Y8z9Y8"),
|
||||
t(ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3q7W3"),
|
||||
t(ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5m4N5"),
|
||||
t(ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"),
|
||||
t(ScrubKind::KeyShape, format!("xxx-ghp_{}", "k2L9".repeat(9)), "k2L9k2L9k2L9"),
|
||||
t(ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2C3"),
|
||||
t(ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7r6T7"),
|
||||
t(ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL"),
|
||||
t(ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF"),
|
||||
t(ScrubKind::KeyShape, "xAKIA0123456789ABCDEF".into(), "0123456789ABCDEF"),
|
||||
t(ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_"),
|
||||
t(ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR"),
|
||||
t(ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5"),
|
||||
t(ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz"),
|
||||
t(ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654"),
|
||||
t(ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ"),
|
||||
t(ScrubKind::PemBlock, "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfake\n-----END RSA PRIVATE KEY-----".into(), "MIIEowIBAAKCAQEAfake"),
|
||||
t(ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake"),
|
||||
t(ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake"),
|
||||
t(ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2"),
|
||||
t(ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345"),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn planted_credentials_never_reach_the_chunk_the_embedder_or_the_journal_in_a_message_or_an_edit() {
|
||||
for (kind, secret, needle) in planted() {
|
||||
for shape in ["see {} now", "see {}", "{}\nnext line"] {
|
||||
let (dirty, clean) = (shape.replace("{}", &secret), "nothing here".to_owned());
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, &dirty), said("$e", 20, &clean), edit("$ed", "@a:s", 30, "$e", &dirty)]);
|
||||
let mut rig = Rig::new();
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(done.is_clean(), "a scrub hit does not make a run unclean");
|
||||
let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::<Vec<_>>());
|
||||
for seen in [stored, rig.embedded().join("\n"), journal.join("\n")] {
|
||||
assert!(!seen.contains(&needle), "{kind}: {secret:?} leaked into {seen}");
|
||||
}
|
||||
assert!(rig.bodies().iter().filter(|body| body.contains("<redacted>")).count() >= 2, "{kind}: both bodies show the replacement");
|
||||
let hits: Vec<(&str, ScrubKind)> = done.scrubbed.iter().map(|hit| (hit.event_id.as_str(), hit.kind)).collect();
|
||||
assert!(hits.contains(&("$m", kind)) && hits.contains(&("$ed", kind)), "{kind}: {hits:?}");
|
||||
assert!(done.scrubbed.iter().all(|hit| hit.room_id == "!r:s"));
|
||||
assert!(journal.contains(&format!("scrubbed: !r:s $m {kind}")), "{journal:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_scrub_meets_edits_redactions_the_body_cut_replays_and_version_bumps() {
|
||||
let token = format!("ghp_{}", "k2L9".repeat(9));
|
||||
let mut history = FakeHistory::default();
|
||||
history.room(
|
||||
"!r:s",
|
||||
vec![
|
||||
said("$clean", 10, "all clean"),
|
||||
said("$dirty", 20, &format!("token {token} inside")),
|
||||
edit("$adds", "@a:s", 30, "$clean", &format!("now with {token}")),
|
||||
edit("$removes", "@a:s", 40, "$dirty", "token removed"),
|
||||
said("$gone", 50, &format!("secret {token}")),
|
||||
redaction("$r", 60, "$gone"),
|
||||
said("$straddle", 70, &format!("{} {token} {}", "x".repeat(CHAT_BODY_CHARS_MAX - 10), "y".repeat(100))),
|
||||
],
|
||||
);
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9")));
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9")));
|
||||
let by_id: BTreeMap<String, String> = rig.points().iter().flat_map(|point| point.payload.messages.iter().map(|message| (message.event_id.clone(), message.body.clone()))).collect();
|
||||
assert_eq!(by_id["$clean"], "now with <redacted>");
|
||||
assert_eq!(by_id["$dirty"], "token removed");
|
||||
assert!(!by_id.contains_key("$gone"));
|
||||
assert!(by_id["$straddle"].chars().count() <= CHAT_BODY_CHARS_MAX && !by_id["$straddle"].contains("ghp_"));
|
||||
assert!(done.scrubbed.iter().any(|hit| hit.event_id == "$adds") && done.scrubbed.iter().any(|hit| hit.event_id == "$gone"));
|
||||
let (ids, texts) = (rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::<Vec<_>>());
|
||||
// Replay from the start, as after a lost cursor: identical ids and text.
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.remove(&cursor_key("!r:s"));
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!((rig.ids(), rig.points().into_iter().map(|point| point.payload.text).collect::<Vec<_>>()), (ids, texts));
|
||||
// An older scrub version re-ingests the room and scrubs it again from the history.
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert(cursor_key("!r:s"), format!("{}.0:$straddle", huginn_mind::redact::SCRUB_VERSION - 1));
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("k2L9k2L9")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_text_is_stored_byte_for_byte() {
|
||||
let mut history = FakeHistory::default();
|
||||
let same = "risk-averse task-force sk- 3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c !abc:s user@example.com ssh://git@host/r.git the bearer of bad news";
|
||||
history.room("!r:s", vec![said("$a", 1, same)]);
|
||||
let mut rig = Rig::new();
|
||||
let done = rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), [same]);
|
||||
assert!(done.scrubbed.is_empty());
|
||||
}
|
||||
|
||||
// ---- steward purge ----
|
||||
|
||||
fn window(from_ms: u64, to_ms: u64) -> PurgeWindow {
|
||||
PurgeWindow { from_ms, to_ms }
|
||||
}
|
||||
|
||||
fn purge_history() -> FakeHistory {
|
||||
let mut history = FakeHistory::default();
|
||||
// t1 = 1000, t2 = 3000: $b at t1 and $c inside are purged, $d at t2 is kept.
|
||||
history.room("!r:s", vec![said("$a", 500, "before"), said("$b", 1000, "PURGED-B"), said("$c", 2000, "PURGED-C"), said("$d", 3000, "after"), said("$e", 4000, "later")]);
|
||||
history.room("!other:s", vec![said("$o1", 1000, "same time other room"), said("$o2", 2000, "same time")]);
|
||||
history
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purge_window_removes_exactly_its_events_and_re_chunks_the_neighbours() {
|
||||
let mut history = purge_history();
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let other_ids: BTreeSet<String> = rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect();
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$b", "$c", "$d", "$e"]);
|
||||
assert!(request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap());
|
||||
let embedded_before = rig.embedded().len();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$e"], "t1 is inside, t2 is not");
|
||||
assert!(rig.embedded()[embedded_before..].iter().all(|text| !text.contains("PURGED")));
|
||||
assert!(rig.points().iter().all(|point| !point.payload.text.contains("PURGED")));
|
||||
assert_eq!(rig.points().iter().filter(|point| point.payload.room_id == "!other:s").map(|point| point.id.clone()).collect::<BTreeSet<_>>(), other_ids);
|
||||
assert_eq!(rig.held_ids("!other:s"), ["$o1", "$o2"]);
|
||||
assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("1000..3000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purged_first_message_changes_the_chunk_id_and_an_emptied_chunk_is_deleted() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1000, "x"), said("$b", 2000, "y"), said("$c", 99_000_000, "far")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids().len(), 2);
|
||||
request_purge(&mut rig.index, "!r:s", window(900, 1500)).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids(), [chunk_id("!r:s", "$b"), chunk_id("!r:s", "$c")].into());
|
||||
request_purge(&mut rig.index, "!r:s", window(99_000_000, 99_000_001)).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purged_window_does_not_come_back() {
|
||||
let mut history = purge_history();
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
request_purge(&mut rig.index, "!r:s", window(1000, 3000)).unwrap();
|
||||
rig.ok(&mut history);
|
||||
let want = ["$a", "$d", "$e"];
|
||||
// New events, an in-window edit of an outside message, a redaction.
|
||||
history.push("!r:s", vec![said("$f", 5000, "new"), edit("$ed", "@a:s", 2500, "$a", "EDIT-IN-WINDOW"), redaction("$rd", 6000, "$e")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"]);
|
||||
assert!(rig.bodies().contains(&"before".to_owned()) && !rig.bodies().iter().any(|body| body.contains("EDIT-IN-WINDOW")), "the in-window edit is dropped and the original stays");
|
||||
// A lost cursor re-ingests from the start; a version bump re-ingests too.
|
||||
for cursor in [None, Some("0.0:$f".to_owned())] {
|
||||
let mut cursors = rig.index.cursors();
|
||||
match cursor {
|
||||
None => cursors.remove(&cursor_key("!r:s")),
|
||||
Some(old) => cursors.insert(cursor_key("!r:s"), old),
|
||||
};
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("PURGED")), "{:?}", rig.held_ids("!r:s"));
|
||||
assert_eq!(rig.held_ids("!r:s"), ["$a", "$d", "$f"], "$e stays redacted by the history; {want:?} before the new events");
|
||||
}
|
||||
// The reader leaves the room and joins again.
|
||||
let mut left = FakeHistory::default();
|
||||
left.room("!other:s", vec![said("$o1", 1000, "x")]);
|
||||
rig.ok(&mut left);
|
||||
assert!(rig.held_ids("!r:s").is_empty() && tombstone(&rig, "!r:s").is_some());
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().all(|body| !body.contains("PURGED")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_purge_of_an_unindexed_window_writes_only_the_tombstone_and_still_holds() {
|
||||
let mut rig = Rig::new();
|
||||
let writes = rig.index.state.lock().unwrap().cursor_writes.len();
|
||||
assert!(request_purge(&mut rig.index, "!never:s", window(10, 20)).unwrap());
|
||||
assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes + 1);
|
||||
assert!(rig.points().is_empty() && rig.embedded().is_empty());
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!never:s", vec![said("$a", 5, "ok"), said("$b", 15, "PURGED"), said("$c", 25, "ok")]);
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.held_ids("!never:s"), ["$a", "$c"]);
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("PURGED")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_repeated_or_overlapping_request_changes_nothing_or_merges_and_the_33rd_window_is_refused() {
|
||||
let mut rig = Rig::new();
|
||||
assert!(request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap());
|
||||
let writes = rig.index.state.lock().unwrap().cursor_writes.len();
|
||||
assert!(!request_purge(&mut rig.index, "!r:s", window(10, 20)).unwrap());
|
||||
assert!(!request_purge(&mut rig.index, "!r:s", window(12, 18)).unwrap());
|
||||
assert_eq!(rig.index.state.lock().unwrap().cursor_writes.len(), writes, "no write for a request the tombstone already holds");
|
||||
assert!(request_purge(&mut rig.index, "!r:s", window(15, 30)).unwrap());
|
||||
assert_eq!(tombstone(&rig, "!r:s").as_deref(), Some("10..30"));
|
||||
for n in 0..(PURGE_WINDOWS_MAX as u64 - 1) {
|
||||
request_purge(&mut rig.index, "!r:s", window(100 + n * 10, 105 + n * 10)).unwrap();
|
||||
}
|
||||
let error = format!("{:#}", request_purge(&mut rig.index, "!r:s", window(10_000, 10_001)).unwrap_err());
|
||||
assert!(error.contains("32"), "{error}");
|
||||
assert!(request_purge(&mut Rig::new().index, "!r:s", window(5, 5)).is_err());
|
||||
}
|
||||
|
||||
// ---- the source stays in its lane ----
|
||||
|
||||
#[test]
|
||||
fn the_ingest_code_is_pure_over_its_ports_and_names_no_credential_shape_no_second_literal_and_no_join_rule() {
|
||||
let sources = [("chat.rs", include_str!("chat.rs")), ("chat_chunk.rs", include_str!("chat_chunk.rs")), ("chat_purge.rs", include_str!("chat_purge.rs")), ("room_scope.rs", include_str!("room_scope.rs"))];
|
||||
let process = ["Command", "::new"].concat();
|
||||
for (name, source) in sources {
|
||||
let code = source.split("#[cfg(test)]").next().unwrap();
|
||||
assert!(!code.contains(&process) && !code.contains("std::process"), "{name}");
|
||||
assert!(!code.to_ascii_lowercase().contains("join_rule"), "{name}");
|
||||
if name != "room_scope.rs" {
|
||||
assert!(!code.to_ascii_lowercase().contains("discord"), "{name}");
|
||||
}
|
||||
for shape in ["sk-", "sk_live_", "ghp_", "github_pat_", "glpat-", "AKIA", "xoxb-", "PRIVATE KEY", "Bearer"] {
|
||||
assert!(!code.contains(shape), "{name} names {shape}");
|
||||
}
|
||||
let defined = usize::from(name == "chat.rs");
|
||||
assert_eq!((code.matches("\"chat:").count(), code.matches("\"chat-purge:").count()), (defined, defined), "{name}: each prefix is defined once");
|
||||
}
|
||||
let chat = include_str!("chat.rs");
|
||||
assert_eq!(chat.matches("scrub(").count(), 2, "scrub is called for a body and for the room name, nowhere else");
|
||||
}
|
||||
|
||||
// ---- redacted edits, redacted originals, late edits ----
|
||||
|
||||
const EDIT_SECRET: &str = "EDIT-SECRET";
|
||||
|
||||
fn nothing_secret_stored(rig: &Rig) {
|
||||
let stored = format!("{:?}", rig.points().iter().map(|point| (&point.payload.messages, &point.payload.text)).collect::<Vec<_>>());
|
||||
assert!(!stored.contains(EDIT_SECRET), "{stored}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redacted_edit_in_the_same_page_reverts_the_message_to_its_earlier_body() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, "original words"), edit("$e1", "@a:s", 20, "$m", "first edit"), edit("$e2", "@a:s", 30, "$m", EDIT_SECRET), redaction("$r", 40, "$e2")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), ["first edit"]);
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains(EDIT_SECRET)));
|
||||
history.room("!q:s", vec![said("$m", 10, "orig"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET), redaction("$r", 30, "$e")]);
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().contains(&"orig".to_owned()));
|
||||
nothing_secret_stored(&rig);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redaction_of_an_edit_a_later_run_stored_rebuilds_the_room_and_the_secret_does_not_return() {
|
||||
let mut history = FakeHistory { prune: true, ..FakeHistory::default() };
|
||||
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e", "@a:s", 20, "$m", EDIT_SECRET)]);
|
||||
history.room("!q:s", vec![said("$q", 10, "quiet")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert!(rig.bodies().iter().any(|body| body == EDIT_SECRET), "the edit applied while it stood");
|
||||
let recreated = rig.index.state.lock().unwrap().recreated;
|
||||
history.push("!r:s", vec![redaction("$r", 30, "$e")]);
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.is_clean());
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.bodies().contains(&"original".to_owned()));
|
||||
assert_eq!(rig.held_ids("!q:s"), ["$q"]);
|
||||
assert_eq!(rig.index.state.lock().unwrap().recreated, recreated);
|
||||
assert_eq!(rig.cursor("!r:s").unwrap(), format!("{}:$r", chat_version()));
|
||||
// The rebuild does not loop: the redaction it replays is not a trigger the second time, and nothing changes.
|
||||
let ids = rig.ids();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.ids(), ids);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_redacted_original_takes_its_live_and_late_edits_with_it_in_a_page_across_runs_and_across_a_rebuild() {
|
||||
let script = |history: &mut FakeHistory| {
|
||||
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", "live edit"), redaction("$re", 25, "$e1"), redaction("$rm", 30, "$m"), edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]);
|
||||
history.prune = true;
|
||||
};
|
||||
// One page.
|
||||
let (mut history, mut rig) = (FakeHistory::default(), Rig::new());
|
||||
script(&mut history);
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.points().is_empty());
|
||||
// Across runs: the original and a live edit stand, then it is redacted, then a late edit arrives.
|
||||
let (mut history, mut rig) = (FakeHistory { prune: true, ..FakeHistory::default() }, Rig::new());
|
||||
history.room("!r:s", vec![said("$m", 10, "original"), edit("$e1", "@a:s", 20, "$m", EDIT_SECRET)]);
|
||||
rig.ok(&mut history);
|
||||
history.push("!r:s", vec![redaction("$rm", 30, "$m")]);
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
history.push("!r:s", vec![edit("$late", "@a:s", 40, "$m", EDIT_SECRET)]);
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.points().is_empty());
|
||||
// Across a rebuild: a lost cursor re-reads the whole history.
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.remove(&cursor_key("!r:s"));
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.ok(&mut history);
|
||||
nothing_secret_stored(&rig);
|
||||
assert!(rig.points().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stranger_cannot_edit_a_message_of_the_same_page() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, "mine"), edit("$e", "@stranger:s", 20, "$m", "hijacked")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), ["mine"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_room_name_is_scrubbed_before_it_is_embedded_or_stored() {
|
||||
let mut history = FakeHistory::default();
|
||||
let token = format!("ghp_{}", "k2L9".repeat(9));
|
||||
history.room("!r:s", vec![said("$m", 10, "hello")]).facts("!r:s", |facts| facts.name = Some(format!("Swarm {token}")));
|
||||
let mut rig = Rig::new();
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert!(rig.embedded().iter().all(|text| !text.contains("k2L9k2L9")));
|
||||
assert!(rig.points().iter().all(|point| !point.payload.text.contains("k2L9k2L9") && point.payload.text.starts_with("Swarm <redacted>")));
|
||||
assert!(done.scrubbed.iter().any(|hit| hit.event_id == "(name)") && journal.iter().all(|line| !line.contains("k2L9k2L9")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_token_split_by_zero_width_characters_leaves_no_tail_in_a_stored_body() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$m", 10, &format!("key sk-{}\u{200b}{} done", "q7W3".repeat(4), "q7W3".repeat(4)))]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
assert_eq!(rig.bodies(), ["key <redacted> done"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retire_goes_on_past_a_failed_delete_and_names_each_failure() {
|
||||
let mut rig = Rig::new();
|
||||
for room in ["!a:s", "!b:s", "!c:s"] {
|
||||
rig.seed(room, "$e", 1);
|
||||
}
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes_for = ["!a:s".to_owned()].into();
|
||||
let done = retire_chat(&mut rig.index, &BTreeSet::new()).unwrap();
|
||||
assert_eq!(done.retired, ["!b:s", "!c:s"]);
|
||||
assert_eq!(done.failed.len(), 1);
|
||||
assert_eq!(done.failed[0].0, "!a:s");
|
||||
assert_eq!(rig.held_ids("!a:s"), ["$e"]);
|
||||
assert!(rig.cursor("!a:s").is_some(), "the cursor stays so the next run finds it again");
|
||||
// The refresh reports it too.
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!z:s", vec![said("$z", 1, "x")]);
|
||||
let done = rig.ok(&mut history);
|
||||
assert!(done.failed.contains(&"!a:s".to_owned()) && !done.is_clean());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_facts_error_on_a_joined_room_keeps_its_points_and_cursor_and_is_reported() {
|
||||
let mut history = FakeHistory::default();
|
||||
history.room("!r:s", vec![said("$a", 1, "x")]);
|
||||
let mut rig = Rig::new();
|
||||
rig.ok(&mut history);
|
||||
let (cursor, points) = (rig.cursor("!r:s"), rig.points());
|
||||
history.facts_fail.insert("!r:s".into());
|
||||
let (done, journal) = rig.run(&mut history);
|
||||
let done = done.unwrap();
|
||||
assert_eq!((rig.cursor("!r:s"), rig.points()), (cursor, points));
|
||||
assert_eq!(done.failed, ["!r:s"]);
|
||||
assert!(done.rooms.iter().any(|report| report.room_id == "!r:s" && report.failure.is_some()));
|
||||
assert!(journal.iter().any(|line| line.contains("!r:s") && line.contains("failed")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_tail_on_equal_ends_is_the_greatest_chunk_id() {
|
||||
let mut rig = Rig::new();
|
||||
let (a, b) = (chunk_id("!r:s", "$a"), chunk_id("!r:s", "$b"));
|
||||
rig.index.upsert_chat(&[
|
||||
ChatPoint { id: a.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$a", 5)]) },
|
||||
ChatPoint { id: b.clone(), vector: vec![1.0; 4], payload: chunk_payload_of("!r:s", &[("$b", 5)]) },
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(rig.index.chat_tail("!r:s").unwrap().unwrap().id, a.max(b));
|
||||
}
|
||||
|
|
@ -3,26 +3,37 @@
|
|||
//! for semantic search. Each point's `corpus` payload (`forge` or `chat`) keeps
|
||||
//! the two apart; a forge search filters `forge` and a chat search `chat`.
|
||||
//!
|
||||
//! Two truths, one writer. The forge is the truth of which repos exist and what
|
||||
//! they hold; the collection is derived from it and can be deleted whenever.
|
||||
//! `refresh` is the only writer of the collection, run by the `huginn-sources`
|
||||
//! oneshot job: first build, incremental refresh, rebuild after a model or
|
||||
//! chunker change, and retirement are all the one routine (delete by filter,
|
||||
//! ingest at a commit, rewrite the cursors). The daemon only reads: `search`
|
||||
//! is its whole part, and nothing in `serve` or the worker calls a write.
|
||||
//! Two truths, one job. The forge is the truth of which repos exist and what
|
||||
//! they hold, Synapse of what the Minds' rooms said; the collection is derived
|
||||
//! from them and can be deleted whenever (but for a steward's purge tombstones, see
|
||||
//! `chat_purge`). `refresh` is the forge's writer and the only creator of the
|
||||
//! collection, run by the `huginn-sources` oneshot job: first build, incremental
|
||||
//! refresh, rebuild after a model or chunker change, and retirement are all the one
|
||||
//! routine (delete by filter, ingest at a commit, rewrite the cursors). `chat::refresh_chat`
|
||||
//! joins it in the same job and the same cursors object, as the only writer of chat points;
|
||||
//! it requires the collection to exist and never recreates it. The daemon only reads: `search`
|
||||
//! and `search_chat` are its whole part, and nothing in `serve` or the worker calls a write.
|
||||
//!
|
||||
//! A cursor is the commit whose chunks the collection holds for a repo. It
|
||||
//! lives in the collection's own metadata, so it is lost with the collection
|
||||
//! A cursor is the commit (or, for a room, the event) whose chunks the collection
|
||||
//! holds. It lives in the collection's own metadata, so it is lost with the collection
|
||||
//! and the two cannot disagree about what is held. It is advanced only after
|
||||
//! the repo's deletes and upserts are acknowledged: it may lag and never leads.
|
||||
//! It is dropped before the repo's first delete, so a run that fails between
|
||||
//! the delete and the upserts leaves no cursor, and the repo is ingested whole.
|
||||
//! The cursors object is written whole by this job alone, once per repo.
|
||||
//! the source's deletes and upserts are acknowledged: it may lag and never leads.
|
||||
//! A repo's is dropped before its first delete, so a run that fails between
|
||||
//! the delete and the upserts leaves no cursor, and the repo is ingested whole; a room's
|
||||
//! chat chunks are written before anything they replace is deleted. The cursors
|
||||
//! object is written whole by this job alone. A key is a repo name, or a chat key
|
||||
//! (`chat::is_chat_key`), never both.
|
||||
|
||||
pub mod chat;
|
||||
pub mod chat_chunk;
|
||||
pub mod chat_purge;
|
||||
pub mod chunk;
|
||||
pub mod forge;
|
||||
pub mod git;
|
||||
pub mod room_scope;
|
||||
|
||||
#[cfg(test)]
|
||||
mod chat_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
|
|
@ -190,6 +201,14 @@ impl ChatFilter {
|
|||
}
|
||||
}
|
||||
|
||||
/// A chat chunk as the index holds it, with its point id: what the ingest job
|
||||
/// reads back to rewrite a chunk.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct StoredChat {
|
||||
pub id: String,
|
||||
pub payload: ChatPayload,
|
||||
}
|
||||
|
||||
/// One chunk a vector search returned and how near it scored.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct SourceRow {
|
||||
|
|
@ -227,6 +246,16 @@ pub trait SourceIndex {
|
|||
/// Up to `limit` chat chunks `filter` keeps, by span start in `order`.
|
||||
/// Reads only `corpus=chat` points; the daemon's.
|
||||
fn chat_scroll(&mut self, filter: &ChatFilter, order: ChatOrder, limit: u32, within: Duration) -> Result<Vec<ChatPayload>>;
|
||||
/// The room's chunk with the greatest `ts_end_ms`, if it has any. The ingest job's alone.
|
||||
fn chat_tail(&mut self, room_id: &str) -> Result<Option<StoredChat>>;
|
||||
/// The room's chunks holding any of `event_ids`. The ingest job's alone.
|
||||
fn chat_chunks_holding(&mut self, room_id: &str, event_ids: &[String]) -> Result<Vec<StoredChat>>;
|
||||
/// The room's chunks whose span reaches into [`from_ms`, `to_ms`): `ts_end_ms >= from_ms`
|
||||
/// and `ts_start_ms < to_ms`. The ingest job's alone.
|
||||
fn chat_chunks_overlapping(&mut self, room_id: &str, from_ms: u64, to_ms: u64) -> Result<Vec<StoredChat>>;
|
||||
/// Deletes a room's chat chunks: those with the point ids `ids`, or all of them when `None`.
|
||||
/// Acknowledged before it returns. Never touches a forge point. The ingest job's alone.
|
||||
fn delete_chat(&mut self, room_id: &str, ids: Option<&[String]>) -> Result<()>;
|
||||
}
|
||||
|
||||
/// A repo the forge lists as indexable.
|
||||
|
|
@ -395,7 +424,7 @@ fn retire<S: SourceIndex>(
|
|||
failed: &mut Vec<String>,
|
||||
) {
|
||||
let listed = |name: &str| catalog.iter().any(|repo| repo.name == name);
|
||||
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !listed(name)).cloned().collect();
|
||||
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !chat::is_chat_key(name) && !listed(name)).cloned().collect();
|
||||
if let Ok(entries) = std::fs::read_dir(mirrors) {
|
||||
for entry in entries.flatten() {
|
||||
if let Some(name) = entry.file_name().to_string_lossy().strip_suffix(".git")
|
||||
|
|
|
|||
184
crates/huginn-daemon/src/sources/room_scope.rs
Normal file
184
crates/huginn-daemon/src/sources/room_scope.rs
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
//! Which joined rooms and which senders are in the chat corpus. The only code
|
||||
//! that decides it, and the only file outside tests that names the bridged
|
||||
//! platform whose content is refused.
|
||||
//!
|
||||
//! Scope (ruling `gamecult-chat:ruling:chat-index-minds-rooms`): the rooms the
|
||||
//! reader user has joined, minus this exclusion. A room's join rule is read by
|
||||
//! nobody (`RoomFacts` has no field for it): a joined invite-only room qualifies
|
||||
//! like any other. The reader roster is the first lock; this file is the second.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::chat::RoomEvent;
|
||||
|
||||
/// The estate rooms by canonical alias, ASCII case-insensitive. Code owns the
|
||||
/// aliases and the rule; the unit supplies the room ids (`Estate`), and either alone excludes.
|
||||
pub const ESTATE_ALIASES: [&str; 3] = ["#users:gamecult.org", "#members:gamecult.org", "#patrons:gamecult.org"];
|
||||
|
||||
/// What the server says about a room, as far as admission needs it. The Matrix
|
||||
/// client fills it from the room's state and decides nothing. It has no `Default`: a
|
||||
/// room whose state could not be read is a port error, never a plain room.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RoomFacts {
|
||||
pub room_id: String,
|
||||
pub alias: Option<String>,
|
||||
pub alt_aliases: Vec<String>,
|
||||
pub name: Option<String>,
|
||||
/// An `m.room.encryption` state event exists.
|
||||
pub encrypted: bool,
|
||||
/// Any `m.room.member` in the state has `is_direct` true.
|
||||
pub direct: bool,
|
||||
/// The state type (`m.bridge` or `uk.half-shot.bridge`) of a bridge event, when the room has one.
|
||||
pub bridge: Option<String>,
|
||||
}
|
||||
|
||||
/// The estate room ids the caller knows (they exist only on the live server): exactly one per
|
||||
/// estate alias, so a job that was handed none, or too few, cannot be built.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Estate {
|
||||
ids: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl Estate {
|
||||
pub fn new(ids: BTreeSet<String>) -> anyhow::Result<Self> {
|
||||
anyhow::ensure!(
|
||||
ids.len() == ESTATE_ALIASES.len() && ids.iter().all(|id| id.starts_with('!')),
|
||||
"the estate is exactly {} room ids ({}), got {}",
|
||||
ESTATE_ALIASES.len(),
|
||||
ESTATE_ALIASES.join(", "),
|
||||
ids.len()
|
||||
);
|
||||
Ok(Self { ids })
|
||||
}
|
||||
|
||||
/// Whether `room_id` is a named estate room. Decided before the room's state is read.
|
||||
pub fn holds_id(&self, room_id: &str) -> bool {
|
||||
self.ids.contains(room_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a joined room, or an event in it, is refused.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Refusal {
|
||||
Estate,
|
||||
Direct,
|
||||
Encrypted,
|
||||
Bridged(String),
|
||||
DiscordSender,
|
||||
DiscordContent,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Refusal {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Estate => formatter.write_str("estate room"),
|
||||
Self::Direct => formatter.write_str("direct room"),
|
||||
Self::Encrypted => formatter.write_str("encrypted room"),
|
||||
Self::Bridged(state_type) => write!(formatter, "bridged room ({state_type})"),
|
||||
Self::DiscordSender => formatter.write_str("discord sender"),
|
||||
Self::DiscordContent => formatter.write_str("discord content"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a joined room may be read at all.
|
||||
pub fn admit_room(facts: &RoomFacts, estate: &Estate) -> Result<(), Refusal> {
|
||||
let estate_alias = |alias: &str| ESTATE_ALIASES.iter().any(|estate| estate.eq_ignore_ascii_case(alias));
|
||||
if estate.holds_id(&facts.room_id) || facts.alias.iter().chain(&facts.alt_aliases).any(|alias| estate_alias(alias)) {
|
||||
return Err(Refusal::Estate);
|
||||
}
|
||||
if facts.direct {
|
||||
return Err(Refusal::Direct);
|
||||
}
|
||||
if facts.encrypted {
|
||||
return Err(Refusal::Encrypted);
|
||||
}
|
||||
if let Some(state_type) = &facts.bridge {
|
||||
return Err(Refusal::Bridged(state_type.clone()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether an event refuses its whole room: a sender whose localpart, lowercased, starts with the
|
||||
/// platform's name after any leading underscores, has it after an underscore (`mautrix_discord`,
|
||||
/// `_bridge_discordbot`), or has it followed by `bot` or an underscore; or content with a top-level key
|
||||
/// naming it. Deterministic identifier rules, not language reading; `discourse` is not it.
|
||||
pub fn refuse_event(event: &RoomEvent) -> Option<Refusal> {
|
||||
let localpart = event.sender.trim_start_matches('@').split(':').next().unwrap_or("").to_ascii_lowercase();
|
||||
let name = ["disc", "ord"].concat();
|
||||
if localpart.trim_start_matches('_').starts_with(&name) || localpart.contains(&format!("_{name}")) || localpart.contains(&format!("{name}bot")) || localpart.contains(&format!("{name}_")) {
|
||||
return Some(Refusal::DiscordSender);
|
||||
}
|
||||
event.content_keys.iter().any(|key| key.to_ascii_lowercase().contains("discord")).then_some(Refusal::DiscordContent)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::chat::EventKind;
|
||||
use super::*;
|
||||
|
||||
fn facts(id: &str) -> RoomFacts {
|
||||
RoomFacts { room_id: id.into(), alias: None, alt_aliases: vec![], name: None, encrypted: false, direct: false, bridge: None }
|
||||
}
|
||||
|
||||
fn estate(ids: &[&str]) -> Estate {
|
||||
Estate::new(ids.iter().chain(["!pad1:s", "!pad2:s", "!pad3:s"].iter()).take(3).map(|id| (*id).to_owned()).collect()).unwrap()
|
||||
}
|
||||
|
||||
fn event(sender: &str, keys: &[&str]) -> RoomEvent {
|
||||
RoomEvent { event_id: "$e".into(), sender: sender.into(), ts_ms: 1, content_keys: keys.iter().map(|key| (*key).into()).collect(), kind: EventKind::Other }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_direct_encrypted_or_bridged_room_is_refused_and_a_plain_room_is_admitted() {
|
||||
let estate = estate(&[]);
|
||||
assert_eq!(admit_room(&RoomFacts { direct: true, ..facts("!a:s") }, &estate), Err(Refusal::Direct));
|
||||
assert_eq!(admit_room(&RoomFacts { encrypted: true, ..facts("!a:s") }, &estate), Err(Refusal::Encrypted));
|
||||
for state_type in ["m.bridge", "uk.half-shot.bridge"] {
|
||||
assert_eq!(admit_room(&RoomFacts { bridge: Some(state_type.into()), ..facts("!a:s") }, &estate), Err(Refusal::Bridged(state_type.into())));
|
||||
}
|
||||
assert_eq!(admit_room(&RoomFacts { alias: Some("#swarm:gamecult.org".into()), name: Some("Swarm".into()), ..facts("!a:s") }, &estate), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_estate_is_refused_by_id_and_by_alias_in_any_case_and_a_lookalike_is_not() {
|
||||
let named = estate(&["!ids:s"]);
|
||||
assert!(named.holds_id("!ids:s") && !named.holds_id("!other:s"));
|
||||
assert_eq!(admit_room(&facts("!ids:s"), &named), Err(Refusal::Estate));
|
||||
for alias in ESTATE_ALIASES.iter().map(|alias| alias.to_string()).chain(["#USERS:Gamecult.Org".to_owned()]) {
|
||||
assert_eq!(admit_room(&RoomFacts { alias: Some(alias.clone()), ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "{alias}");
|
||||
assert_eq!(admit_room(&RoomFacts { alt_aliases: vec!["#a:s".into(), alias.clone()], ..facts("!x:s") }, &estate(&[])), Err(Refusal::Estate), "alt {alias}");
|
||||
}
|
||||
for lookalike in ["#users-guide:gamecult.org", "#patrons-club:gamecult.org", "#users:gamecult.org.evil", "#users:other.org"] {
|
||||
assert_eq!(admit_room(&RoomFacts { alias: Some(lookalike.into()), ..facts("!x:s") }, &estate(&[])), Ok(()), "{lookalike}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bridged_platform_sender_or_content_key_refuses_and_a_lookalike_does_not() {
|
||||
for sender in ["@discord_123:s", "@_discord_9:s", "@__DiscordBot:s", "@discordbot:s", "@mautrix_discord:s", "@_bridge_discordbot:s", "@a_discord:s"] {
|
||||
assert_eq!(refuse_event(&event(sender, &[])), Some(Refusal::DiscordSender), "{sender}");
|
||||
}
|
||||
for key in ["fi.mau.discord", "uk.half-shot.discord", "X.DISCORD.y"] {
|
||||
assert_eq!(refuse_event(&event("@a:s", &["body", key])), Some(Refusal::DiscordContent), "{key}");
|
||||
}
|
||||
for sender in ["@discourse:s", "@disco:s", "@eureka:s", "@x:discord.com"] {
|
||||
assert_eq!(refuse_event(&event(sender, &["discourse", "body"])), None, "{sender}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_estate_needs_exactly_its_room_ids() {
|
||||
for ids in [vec![], vec!["!a:s"], vec!["!a:s", "!b:s", "!c:s", "!d:s"], vec!["!a:s", "!b:s", "notaroom"]] {
|
||||
assert!(Estate::new(ids.iter().map(|id| (*id).to_owned()).collect()).is_err(), "{ids:?}");
|
||||
}
|
||||
assert!(Estate::new(["!a:s", "!b:s", "!c:s"].map(str::to_owned).into()).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_room_facts_carries_no_join_rule() {
|
||||
let source = include_str!("room_scope.rs");
|
||||
let code = &source[..source.find("#[cfg(test)]").unwrap()];
|
||||
assert!(!code.contains(&["join", "_rule"].concat()));
|
||||
}
|
||||
}
|
||||
|
|
@ -1155,3 +1155,24 @@ fn a_window_asks_the_index_only_and_needs_no_embedder() {
|
|||
let signature = &signature[..signature.find('{').unwrap()];
|
||||
assert!(!signature.contains("Embedder"), "{signature}");
|
||||
}
|
||||
|
||||
/// A chat cursor and a purge tombstone share the forge's cursors object; the
|
||||
/// forge retire never reads either as a repo name, and still retires a repo.
|
||||
#[test]
|
||||
fn chat_cursor_and_purge_tombstone_keys_survive_the_forge_retire() {
|
||||
let (keep, gone) = (Remote::new(), Remote::new());
|
||||
keep.commit(&[("a.rs", b"a\n")], &[]);
|
||||
gone.commit(&[("b.rs", b"b\n")], &[]);
|
||||
let mut rig = Rig::new();
|
||||
rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("Gone", &gone)]));
|
||||
let mut cursors = rig.index.cursors();
|
||||
cursors.insert("chat:!r:s".into(), "1:$e".into());
|
||||
cursors.insert("chat-purge:!r:s".into(), "1..2".into());
|
||||
rig.index.set_cursors(&cursors).unwrap();
|
||||
rig.index.upsert_chat(&[chat_point("!r:s", None, 0, 4, &[("$e", 5)])]).unwrap();
|
||||
rig.converged(&mut FakeForge::of(&[("Keep", &keep)]));
|
||||
let after = rig.index.cursors();
|
||||
assert_eq!(after.keys().map(String::as_str).collect::<Vec<_>>(), ["Keep", "chat-purge:!r:s", "chat:!r:s"], "Gone retired; both chat keys kept");
|
||||
assert!(rig.index.state.lock().unwrap().deletes.iter().all(|(repo, _)| !repo.starts_with("chat")));
|
||||
assert_eq!(rig.index.state.lock().unwrap().chat.len(), 1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ mod rows;
|
|||
pub mod oracle;
|
||||
pub mod query;
|
||||
pub mod receipt;
|
||||
pub mod redact;
|
||||
pub mod refusal;
|
||||
pub mod store;
|
||||
pub mod wire;
|
||||
|
|
|
|||
514
crates/huginn-mind/src/redact.rs
Normal file
514
crates/huginn-mind/src/redact.rs
Normal file
|
|
@ -0,0 +1,514 @@
|
|||
//! Credential redaction: the only code that knows the shape of a secret in text.
|
||||
//!
|
||||
//! `redact_url_userinfo` redacts URL credentials (the Self run's journal and reply,
|
||||
//! the Persona's structured output). `scrub` is chat ingest's door: every message body
|
||||
//! passes it before it is chunked, embedded or stored, and it also runs the URL rule.
|
||||
//! Shapes only; nothing here reads meaning, and a hit is a kind, never a value.
|
||||
|
||||
/// The one redaction of URL credentials. Call it at each place a string the CLI produced enters the journal, her mind or chat: today
|
||||
/// the Self result (`result_text`: journal line and Reply) and the structured output of a Persona or Sleep result
|
||||
/// (`persona::structured_output`, string by string). It does not cover claude's stderr (inherited) or the CLI's transcript files.
|
||||
///
|
||||
/// A forge token rides in a remote URL, so every `scheme://` and every protocol-relative `//` opens an authority, read as git, curl and
|
||||
/// RFC 3986 read it: it ends at the first `/`, `?`, `#`, backslash, quote, backtick, parenthesis, bracket, angle bracket or ASCII whitespace
|
||||
/// (`is_authority_end`). Its userinfo is everything before the authority's LAST `@` (a raw `@` in a password still redacts), and non-ASCII
|
||||
/// characters, NBSP and U+2028 included, stay inside it. The userinfo becomes `<redacted>`: `scheme://<redacted>@host`. With a colon it is
|
||||
/// always redacted; without one it is a bare token on http, https, ftp, ftps, ws, wss, any `+http`/`+https` scheme (`git+https`), protocol-relative `//` and an unparseable scheme, and a plain
|
||||
/// username on any other scheme (`ssh://git@host`), which stays. A `@` after the authority (`github.com/@user`, `pkg@1.2.3`) is path, not
|
||||
/// userinfo, and is left alone. Scheme-less credentials are covered only in the shape `word:secret@host.tld/` at the start of an
|
||||
/// ASCII-whitespace-delimited word. Not covered: a password holding `/ ? # ` a quote or ASCII whitespace (no parser reads it as userinfo),
|
||||
/// scp-style `user:secret@host:path`, tokens in query strings, claude's stderr, the kept transcripts. Syntax only; no reading of meaning.
|
||||
pub fn redact_url_userinfo(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut rest = text;
|
||||
while let Some(at) = rest.find("//") {
|
||||
let (before, after) = rest.split_at(at);
|
||||
let body = &after[2..];
|
||||
out.push_str(before);
|
||||
out.push_str("//");
|
||||
// The scheme is the ASCII run before `:`. Without a colon, `//` opens an authority only at a word start (`a//b@c` is not one).
|
||||
let scheme = match before.strip_suffix(':') {
|
||||
// Only ASCII bytes match, so the byte after the last non-match is a character boundary.
|
||||
Some(head) => head[head.bytes().rposition(|c| !(c.is_ascii_alphanumeric() || matches!(c, b'+' | b'-' | b'.'))).map_or(0, |i| i + 1)..].to_ascii_lowercase(),
|
||||
None if before.chars().next_back().is_none_or(|c| is_break(c) || matches!(c, '"' | '\'' | '`' | '(' | '[' | '{' | '<' | '=' | ',')) => String::new(),
|
||||
None => {
|
||||
rest = body;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let end = body.find(is_authority_end).unwrap_or(body.len());
|
||||
let authority = &body[..end];
|
||||
match authority.rfind('@') {
|
||||
Some(i) if i > 0 && (authority[..i].contains(':') || carries_bare_token(&scheme)) => {
|
||||
out.push_str("<redacted>");
|
||||
out.push_str(&authority[i..]);
|
||||
}
|
||||
_ => out.push_str(authority),
|
||||
}
|
||||
rest = &body[end..];
|
||||
}
|
||||
out.push_str(rest);
|
||||
redact_schemeless(&out)
|
||||
}
|
||||
|
||||
/// Schemes whose lone userinfo is a token, not a username: web and ftp schemes, `ws`/`wss`, and `x+http`/`x+https` wrappers; `scheme` is lowercase.
|
||||
fn carries_bare_token(scheme: &str) -> bool {
|
||||
matches!(scheme, "" | "http" | "https" | "ftp" | "ftps" | "ws" | "wss") || scheme.ends_with("+http") || scheme.ends_with("+https")
|
||||
}
|
||||
|
||||
/// ASCII whitespace; NBSP, U+2028 and other Unicode whitespace are not, so a secret holding them is redacted whole.
|
||||
fn is_break(c: char) -> bool {
|
||||
matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c')
|
||||
}
|
||||
|
||||
/// The end of a URL's authority.
|
||||
fn is_authority_end(c: char) -> bool {
|
||||
is_break(c) || matches!(c, '/' | '?' | '#' | '\\' | '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | '<' | '>')
|
||||
}
|
||||
|
||||
/// `word:secret@host.tld/` at the start of a whitespace-delimited word that holds no `//` becomes `<redacted>@host.tld/`.
|
||||
fn redact_schemeless(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut word = String::new();
|
||||
let flush = |word: &mut String, out: &mut String| {
|
||||
out.push_str(&redact_word(word));
|
||||
word.clear();
|
||||
};
|
||||
for c in text.chars() {
|
||||
if is_break(c) {
|
||||
flush(&mut word, &mut out);
|
||||
out.push(c);
|
||||
} else {
|
||||
word.push(c);
|
||||
}
|
||||
}
|
||||
flush(&mut word, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn redact_word(word: &str) -> String {
|
||||
let shaped = (|| {
|
||||
if word.contains("//") {
|
||||
return None;
|
||||
}
|
||||
let at = word.rfind('@')?;
|
||||
let colon = word.find(':')?;
|
||||
let (user, host) = (&word[..colon], &word[at + 1..]);
|
||||
let host_name = &host[..host.find('/')?];
|
||||
let user_ok = !user.is_empty() && user.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'));
|
||||
let host_ok = host_name.contains('.') && host_name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-'));
|
||||
(colon + 1 < at && user_ok && host_ok).then_some(at)
|
||||
})();
|
||||
match shaped {
|
||||
Some(at) => format!("<redacted>{}", &word[at..]),
|
||||
None => word.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The version of the scrub's rules. Part of the chat version: a bump makes every
|
||||
/// room re-ingest from the room's history and scrub it again.
|
||||
pub const SCRUB_VERSION: u32 = 2;
|
||||
|
||||
/// What replaces a secret.
|
||||
pub const REDACTED: &str = "<redacted>";
|
||||
|
||||
/// The shape of secret a scrub found. Never its value.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum ScrubKind {
|
||||
UrlUserinfo,
|
||||
Bearer,
|
||||
AuthorizationHeader,
|
||||
KeyShape,
|
||||
Jwt,
|
||||
PemBlock,
|
||||
AccessToken,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ScrubKind {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(match self {
|
||||
Self::UrlUserinfo => "url-userinfo",
|
||||
Self::Bearer => "bearer",
|
||||
Self::AuthorizationHeader => "authorization-header",
|
||||
Self::KeyShape => "key-shape",
|
||||
Self::Jwt => "jwt",
|
||||
Self::PemBlock => "pem-block",
|
||||
Self::AccessToken => "access-token",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A body with its secrets replaced by `<redacted>`, and the kinds found, one per secret.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Scrubbed {
|
||||
pub text: String,
|
||||
pub hits: Vec<ScrubKind>,
|
||||
}
|
||||
|
||||
/// Zero-width characters a secret could be split with.
|
||||
fn zero_width(c: char) -> bool {
|
||||
matches!(c, '\u{200b}' | '\u{200c}' | '\u{200d}' | '\u{2060}' | '\u{feff}' | '\u{00ad}' | '\u{180e}')
|
||||
}
|
||||
|
||||
/// Replaces credential shapes in `text`: a PEM or PGP private key block, an `authorization:` header
|
||||
/// line, `Bearer`/`Basic`/`token` followed by a long token (blank or colon between), a vendor key
|
||||
/// shape (Matrix tokens included), an `access_token` query or JSON value, a JWT, and URL userinfo.
|
||||
/// Each hit becomes `<redacted>`. Token shapes only; ordinary prose, SHAs, UUIDs and ids pass byte for byte.
|
||||
/// Zero-width characters are removed before matching; when anything matched, the returned text is the
|
||||
/// stripped one, so a token split by them leaves no tail, and when nothing matched the text is returned as it came (emoji joiners stay).
|
||||
pub fn scrub(text: &str) -> Scrubbed {
|
||||
let mut hits = Vec::new();
|
||||
let mut stripped: String = text.chars().filter(|c| !zero_width(*c)).collect();
|
||||
let passes: [(ScrubKind, fn(&str, usize) -> Option<usize>); 6] = [
|
||||
(ScrubKind::PemBlock, pem_block),
|
||||
(ScrubKind::AuthorizationHeader, authorization_line),
|
||||
(ScrubKind::AccessToken, access_token),
|
||||
(ScrubKind::Bearer, bearer_token),
|
||||
(ScrubKind::KeyShape, key_shape),
|
||||
(ScrubKind::Jwt, jwt),
|
||||
];
|
||||
for (kind, find) in passes {
|
||||
stripped = replace_matches(&stripped, kind, find, &mut hits);
|
||||
}
|
||||
let urls = redact_url_userinfo(&stripped);
|
||||
let (before, after) = (stripped.matches(REDACTED).count(), urls.matches(REDACTED).count());
|
||||
hits.extend(std::iter::repeat_n(ScrubKind::UrlUserinfo, after.saturating_sub(before)));
|
||||
if hits.is_empty() { Scrubbed { text: text.to_owned(), hits } } else { Scrubbed { text: urls, hits } }
|
||||
}
|
||||
|
||||
fn replace_matches(text: &str, kind: ScrubKind, find: fn(&str, usize) -> Option<usize>, hits: &mut Vec<ScrubKind>) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut at = 0;
|
||||
while at < text.len() {
|
||||
match find(text, at) {
|
||||
Some(end) if end > at => {
|
||||
out.push_str(REDACTED);
|
||||
hits.push(kind);
|
||||
at = end;
|
||||
}
|
||||
_ => {
|
||||
let next = at + text[at..].chars().next().map_or(1, char::len_utf8);
|
||||
out.push_str(&text[at..next]);
|
||||
at = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether `at` starts a word: the byte before it is not an ASCII letter or digit.
|
||||
fn word_start(text: &str, at: usize) -> bool {
|
||||
at == 0 || !text.as_bytes()[at - 1].is_ascii_alphanumeric()
|
||||
}
|
||||
|
||||
/// The end of the run of bytes at `from` that `member` accepts.
|
||||
fn run_end(text: &str, from: usize, member: impl Fn(u8) -> bool) -> usize {
|
||||
from + text.as_bytes()[from..].iter().take_while(|byte| member(**byte)).count()
|
||||
}
|
||||
|
||||
fn starts_with_ignore_case(text: &str, at: usize, word: &str) -> bool {
|
||||
text.as_bytes().get(at..at + word.len()).is_some_and(|held| held.eq_ignore_ascii_case(word.as_bytes()))
|
||||
}
|
||||
|
||||
fn token_char(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'~' | b'+' | b'/' | b'=' | b'-')
|
||||
}
|
||||
|
||||
fn key_char(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')
|
||||
}
|
||||
|
||||
/// `-----BEGIN ... PRIVATE KEY-----` through its `-----END ... PRIVATE KEY-----`, or the text's end.
|
||||
fn pem_block(text: &str, at: usize) -> Option<usize> {
|
||||
let rest = &text[at..];
|
||||
let label = rest.strip_prefix("-----BEGIN ")?;
|
||||
let words = run_end(label, 0, |byte| byte.is_ascii_uppercase() || byte == b' ');
|
||||
(label[..words].ends_with("PRIVATE KEY") || label[..words].ends_with("PRIVATE KEY BLOCK")).then_some(())?;
|
||||
label[words..].starts_with("-----").then_some(())?;
|
||||
let body_from = at + "-----BEGIN ".len() + words + 5;
|
||||
let end = text[body_from..].find("-----END ").and_then(|end| {
|
||||
let name = body_from + end + "-----END ".len();
|
||||
let words = run_end(text, name, |byte| byte.is_ascii_uppercase() || byte == b' ');
|
||||
text[words..].starts_with("-----").then_some(words + 5)
|
||||
});
|
||||
Some(end.unwrap_or(text.len()))
|
||||
}
|
||||
|
||||
/// `authorization:` in any case at a word start, through the end of the line.
|
||||
fn authorization_line(text: &str, at: usize) -> Option<usize> {
|
||||
(word_start(text, at) && starts_with_ignore_case(text, at, "authorization:")).then_some(())?;
|
||||
let from = at + "authorization:".len();
|
||||
let end = run_end(text, from, |byte| byte != b'\n' && byte != b'\r');
|
||||
(!text[from..end].trim().is_empty()).then_some(end)
|
||||
}
|
||||
|
||||
/// `Bearer`, `Basic` or `token`, blanks, then 16 or more token characters.
|
||||
fn bearer_token(text: &str, at: usize) -> Option<usize> {
|
||||
word_start(text, at).then_some(())?;
|
||||
let word = ["bearer", "basic", "token"].into_iter().find(|word| starts_with_ignore_case(text, at, word))?;
|
||||
let blanks = at + word.len();
|
||||
let colon = usize::from(text.as_bytes().get(blanks) == Some(&b':'));
|
||||
let value = run_end(text, blanks + colon, |byte| byte == b' ' || byte == b'\t');
|
||||
(value > blanks).then_some(())?;
|
||||
let end = run_end(text, value, token_char);
|
||||
(end - value >= 16).then_some(end)
|
||||
}
|
||||
|
||||
/// A vendor key: the prefix, then a long enough run of the key alphabet.
|
||||
fn key_shape(text: &str, at: usize) -> Option<usize> {
|
||||
let rest = &text[at..];
|
||||
let by_prefix = [("sk-", 16), ("sk_live_", 16), ("pk_live_", 16), ("github_pat_", 16), ("glpat-", 16), ("syt_", 16), ("mct_", 16)];
|
||||
let tokens = ["ghp_", "gho_", "ghu_", "ghs_", "ghr_"].into_iter().map(|prefix| (prefix, 16));
|
||||
for (prefix, least) in by_prefix.into_iter().chain(tokens) {
|
||||
// Only the generic prefix needs a word start (`risk-averse`, `task-force`); the distinctive ones are found glued to anything.
|
||||
if rest.starts_with(prefix) && (prefix != "sk-" || word_start(text, at)) {
|
||||
let end = run_end(text, at + prefix.len(), key_char);
|
||||
return (end - at - prefix.len() >= least).then_some(end);
|
||||
}
|
||||
}
|
||||
for prefix in ["xoxa-", "xoxb-", "xoxp-", "xoxr-", "xoxs-"] {
|
||||
if rest.starts_with(prefix) {
|
||||
let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_alphanumeric() || byte == b'-');
|
||||
return (end - at - prefix.len() >= 16).then_some(end);
|
||||
}
|
||||
}
|
||||
for prefix in ["AKIA", "ASIA"] {
|
||||
if rest.starts_with(prefix) {
|
||||
let end = run_end(text, at + prefix.len(), |byte| byte.is_ascii_uppercase() || byte.is_ascii_digit());
|
||||
return (end - at - prefix.len() >= 16).then_some(end);
|
||||
}
|
||||
}
|
||||
if rest.starts_with("AIza") {
|
||||
let end = run_end(text, at + 4, key_char);
|
||||
return (end - at - 4 >= 35).then_some(end);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// `access_token` as a query parameter or JSON member, through its value (8 or more characters).
|
||||
fn access_token(text: &str, at: usize) -> Option<usize> {
|
||||
starts_with_ignore_case(text, at, "access_token").then_some(())?;
|
||||
let mut at = at + "access_token".len();
|
||||
let skip = |at: usize, set: &[u8]| run_end(text, at, |byte| set.contains(&byte));
|
||||
at = skip(at, b"\"'");
|
||||
at = skip(at, b" \t");
|
||||
matches!(text.as_bytes().get(at), Some(b'=' | b':')).then_some(())?;
|
||||
at = skip(at + 1, b" \t");
|
||||
at = skip(at, b"\"'");
|
||||
let end = run_end(text, at, |byte| !byte.is_ascii_whitespace() && !matches!(byte, b'&' | b'"' | b'\'' | b',' | b'}' | b';' | b'#'));
|
||||
(end - at >= 8).then_some(end)
|
||||
}
|
||||
|
||||
/// Three base64url segments, the first starting `eyJ`.
|
||||
fn jwt(text: &str, at: usize) -> Option<usize> {
|
||||
text[at..].starts_with("eyJ").then_some(())?;
|
||||
let segment = |from: usize| run_end(text, from, key_char);
|
||||
let first = segment(at);
|
||||
let mut end = first;
|
||||
for _ in 0..2 {
|
||||
(text.as_bytes().get(end) == Some(&b'.')).then_some(())?;
|
||||
let next = segment(end + 1);
|
||||
(next > end + 1).then_some(())?;
|
||||
end = next;
|
||||
}
|
||||
Some(end)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const CANARY: &str = "CANARY-NOT-A-REAL-TOKEN-0000";
|
||||
|
||||
#[test]
|
||||
fn url_userinfo_is_redacted_where_it_is_a_secret_and_kept_where_it_is_a_username() {
|
||||
let said = format!(
|
||||
"a https://user:{CANARY}@host.example/r.git b https://{CANARY}@host.example/r.git c ssh://u:{CANARY}@h/x d http://{CANARY}@h e HTTPS://x:{CANARY}@h \
|
||||
f https://u:p@ss{CANARY}@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://u:{CANARY}@h) k ://@x"
|
||||
);
|
||||
let want = "a https://<redacted>@host.example/r.git b https://<redacted>@host.example/r.git c ssh://<redacted>@h/x d http://<redacted>@h e HTTPS://<redacted>@h \
|
||||
f https://<redacted>@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://<redacted>@h) k ://@x";
|
||||
assert_eq!(redact_url_userinfo(&said), want);
|
||||
assert!(!redact_url_userinfo(&said).contains(CANARY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_secret_is_redacted_whole_by_the_authority_rule_and_commonplace_urls_are_left_alone() {
|
||||
let r = redact_url_userinfo;
|
||||
// Raw @, percent-encoding and non-ASCII (NBSP, U+2028) stay inside userinfo.
|
||||
for secret in ["pa@ssT", "pa%40ssT", "pa\u{a0}ssT", "pa\u{2028}ssT", "pa:ssT", "\u{e9}\u{65e5}ssT"] {
|
||||
let out = r(&format!("x https://u:{secret}@h/x"));
|
||||
assert_eq!(out, "x https://<redacted>@h/x", "{secret:?}");
|
||||
}
|
||||
// The authority ends at / ? # \\ quotes, brackets and ASCII whitespace: no parser reads those as userinfo, so what follows is path.
|
||||
assert_eq!(r("https://u:pa/ssT@h/x"), "https://u:pa/ssT@h/x");
|
||||
assert_eq!(r("https://u:pa ssT@h/x"), "https://u:pa ssT@h/x");
|
||||
assert_eq!(r("https://u:pa?ssT@h"), "https://u:pa?ssT@h");
|
||||
// Ordinary URLs with @ in the path are byte-identical.
|
||||
for same in [
|
||||
"https://github.com/@user/repo",
|
||||
"https://medium.com/@name/post-1",
|
||||
"https://registry.npmjs.org/@scope/pkg",
|
||||
"https://h/pkg@1.2.3",
|
||||
"a//b@c",
|
||||
"see //cdn.h/@x/y and https://h/?mail=a@b.c#f@g",
|
||||
"https://h:8080/p git@github.com:org/r.git ssh://git@host/r.git ssh://git@h:22/r git://git@h/x",
|
||||
] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
// Compact JSON: only the credentialed URL changes.
|
||||
let json = format!(r#"{{"a":"https://u:{CANARY}@h/x","b":"https://h2/@v","c":"https://h3/p@1"}}"#);
|
||||
assert_eq!(r(&json), r#"{"a":"https://<redacted>@h/x","b":"https://h2/@v","c":"https://h3/p@1"}"#);
|
||||
assert_eq!(r("a/https://u:T@h/x,https://v:T@h2"), "a/https://<redacted>@h/x,https://<redacted>@h2");
|
||||
assert_eq!(r("HTTPS://TOKEN@h"), "HTTPS://<redacted>@h");
|
||||
assert_eq!(r("[l](https://T@h/x) <https://T@h> `https://u:T@h`"), "[l](https://<redacted>@h/x) <https://<redacted>@h> `https://<redacted>@h`");
|
||||
assert_eq!(r("a //u:T@host/x b //T@host c \"//T@h\" (//T@h)"), "a //<redacted>@host/x b //<redacted>@host c \"//<redacted>@h\" (//<redacted>@h)");
|
||||
// Token-bearing schemes beyond http: +http/+https wrappers and ws/wss redact a bare token; git+ssh and ssh keep their username.
|
||||
assert_eq!(r("git+https://T@github.com/a/b GIT+HTTPS://T@h wss://T@h/ ws://T@h/"), "git+https://<redacted>@github.com/a/b GIT+HTTPS://<redacted>@h wss://<redacted>@h/ ws://<redacted>@h/");
|
||||
for same in ["git+ssh://git@host/x", "ssh://git@host/x"] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
// Scheme-less: only word:secret@host.tld/ . Each guard of the shape has its own case.
|
||||
assert_eq!(r("user:T@host.tld/path"), "<redacted>@host.tld/path");
|
||||
for same in ["a:b@c", "a:b@host/x", "a:b@ho_st.tld/x", "u$er:T@host.tld/x", "a:@host.tld/x", "mail me@x.org/ and :T@host.tld/x"] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multibyte_text_never_panics_the_redaction() {
|
||||
let r = redact_url_userinfo;
|
||||
for case in ["see \u{201c}https://h/x", "\u{2192}https://h/x", "docs\u{ff1a}https://h/x", "\u{e9}://x", "\u{65e5}\u{672c}://u:T@h", "https://x/\u{e9}://", "\u{e9}//u:T@h", "\u{e9}:u:T@h.tld/"] {
|
||||
let _ = r(case);
|
||||
}
|
||||
assert_eq!(r("\u{65e5}\u{672c}://u:T@h"), "\u{65e5}\u{672c}://<redacted>@h");
|
||||
assert_eq!(r("\u{2192}https://u:T@h/x"), "\u{2192}https://<redacted>@h/x");
|
||||
let seed = "\u{201c}\u{2192}\u{ff1a}\u{e9}\u{65e5}\u{a0}https://u:T@h/x//y:z@a.b/ ://@ //";
|
||||
let chars: Vec<char> = seed.chars().collect();
|
||||
for start in 0..=chars.len() {
|
||||
for end in start..=chars.len() {
|
||||
let _ = r(&chars[start..end].iter().collect::<String>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// Every shape is built here from fake parts; none is a real credential.
|
||||
fn secrets() -> Vec<(ScrubKind, String, String)> {
|
||||
let pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAfakefakefake\n-----END RSA PRIVATE KEY-----".to_owned();
|
||||
vec![
|
||||
(ScrubKind::AuthorizationHeader, format!("Authorization: Bearer {}", "a1B2".repeat(8)), "a1B2a1B2".into()),
|
||||
(ScrubKind::Bearer, format!("Bearer {}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()),
|
||||
(ScrubKind::Bearer, format!("Basic {}", "dXNlcjpwYXNz".repeat(2)), "dXNlcjpwYXNzdXNl".into()),
|
||||
(ScrubKind::KeyShape, format!("sk-{}", "q7W3".repeat(8)), "q7W3q7W3".into()),
|
||||
(ScrubKind::KeyShape, format!("sk_live_{}", "m4N5".repeat(6)), "m4N5m4N5".into()),
|
||||
(ScrubKind::KeyShape, format!("ghp_{}", "k2L9".repeat(9)), "k2L9k2L9".into()),
|
||||
(ScrubKind::KeyShape, format!("github_pat_{}_{}", "A1b2C3".repeat(4) + "ab", "d4E5f6".repeat(9) + "abcde"), "A1b2C3A1b2".into()),
|
||||
(ScrubKind::KeyShape, format!("glpat-{}", "r6T7".repeat(5)), "r6T7r6T7".into()),
|
||||
(ScrubKind::KeyShape, "xoxb-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "AKIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()),
|
||||
(ScrubKind::KeyShape, format!("AIza{}", "Sy1_".repeat(9)), "Sy1_Sy1_Sy1_".into()),
|
||||
(ScrubKind::Jwt, "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.c2lnbmF0dXJlZmFrZQ".into(), "c2lnbmF0dXJlZmFrZQ".into()),
|
||||
(ScrubKind::PemBlock, pem, "MIIEowIBAAKCAQEAfake".into()),
|
||||
(ScrubKind::PemBlock, "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANfakeNOEND".into(), "MIIBVQIBADANfake".into()),
|
||||
(ScrubKind::KeyShape, format!("syt_{}_{}_{}", "YWxpY2U", "aBcDeFgHiJkLmNoPqR", "0a1B2c"), "aBcDeFgHiJkLmNoPqR".into()),
|
||||
(ScrubKind::KeyShape, format!("mct_{}", "w4X5".repeat(8)), "w4X5w4X5w4X5".into()),
|
||||
(ScrubKind::KeyShape, "xoxa-2-123456789012-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "xoxp-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "xoxr-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "xoxs-123456789012-987654321098-aBcDeFgHiJkLmNoP".into(), "aBcDeFgHiJkL".into()),
|
||||
(ScrubKind::KeyShape, "ASIA0123456789ABCDEF".into(), "0123456789ABCDEF".into()),
|
||||
(ScrubKind::KeyShape, "gho_".to_owned() + &"p1Q2".repeat(9), "p1Q2p1Q2p1Q2".into()),
|
||||
(ScrubKind::KeyShape, "pk_live_".to_owned() + &"e3R4".repeat(6), "e3R4e3R4e3R4".into()),
|
||||
(ScrubKind::AccessToken, "https://h/_matrix/client/v3/sync?access_token=abcDEF123456xyz&since=s1".into(), "abcDEF123456xyz".into()),
|
||||
(ScrubKind::AccessToken, "{\"access_token\": \"tokenValue987654\", \"user\": 1}".into(), "tokenValue987654".into()),
|
||||
(ScrubKind::Bearer, format!("bearer:{}", "z9Y8x7".repeat(4)), "z9Y8x7z9Y8".into()),
|
||||
(ScrubKind::Bearer, format!("token: {}", "m1N2o3".repeat(4)), "m1N2o3m1N2".into()),
|
||||
(ScrubKind::PemBlock, "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFakeFakeFake\n-----END PGP PRIVATE KEY BLOCK-----".into(), "lQOYBFakeFake".into()),
|
||||
(ScrubKind::UrlUserinfo, "https://user:hunter2hunter2@host.example/r.git".into(), "hunter2hunter2".into()),
|
||||
(ScrubKind::UrlUserinfo, "https://tokenvalue12345@host.example/".into(), "tokenvalue12345".into()),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_planted_credential_is_replaced_mid_line_and_at_line_end_with_its_kind() {
|
||||
for (kind, secret, distinct) in secrets() {
|
||||
for body in [format!("before {secret} after"), format!("before {secret}"), format!("{secret}\nnext line")] {
|
||||
let out = scrub(&body);
|
||||
assert!(!out.text.contains(&distinct), "{kind}: {}", out.text);
|
||||
assert!(out.text.contains(REDACTED), "{kind}: {}", out.text);
|
||||
assert!(out.hits.contains(&kind), "{kind} not in {:?} for {body:?}", out.hits);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pem_block_without_its_end_runs_to_the_end_of_the_body_and_one_with_it_stops_there() {
|
||||
let open = scrub("see -----BEGIN EC PRIVATE KEY-----\nAAAA\nBBBB").text;
|
||||
assert_eq!(open, "see <redacted>");
|
||||
let closed = scrub("-----BEGIN PRIVATE KEY-----\nAAAA\n-----END PRIVATE KEY----- and then prose").text;
|
||||
assert_eq!(closed, "<redacted> and then prose");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_authorization_header_goes_to_the_end_of_its_line_only() {
|
||||
assert_eq!(scrub("hi\nAUTHORIZATION: Token abc\nbye").text, "hi\n<redacted>\nbye");
|
||||
assert_eq!(scrub("Authorization: x").hits, [ScrubKind::AuthorizationHeader]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_text_passes_byte_for_byte_and_reports_no_hit() {
|
||||
for same in [
|
||||
"risk-averse", "task-force", "sk-", "sk-short", "3f2b8c1d9e7a4b5c6d0e1f2a3b4c5d6e7f8a9b0c",
|
||||
"123e4567-e89b-12d3-a456-426614174000", "!abc:s", "$Ab3dEf-9_zQ", "user@example.com", "ssh://git@host/r.git",
|
||||
"https://github.com/@user", "pkg@1.2.3", "the bearer of bad news", "Authorization is a header",
|
||||
"padding like dGVzdA== in prose", "```\nlet x = 1;\n```", "the BEGIN PRIVATE KEY header", "a token of thanks",
|
||||
"AKIAshort", "ghp_short", "eyJ is a prefix", "naïve café — 日本語 with Bearer in it",
|
||||
] {
|
||||
let out = scrub(same);
|
||||
assert_eq!(out.text, same);
|
||||
assert!(out.hits.is_empty(), "{same}: {:?}", out.hits);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multibyte_text_around_a_secret_never_panics_the_scrub() {
|
||||
let body = format!("日本 \u{a0}é Bearer {} → ünï", "A1b2".repeat(6));
|
||||
assert_eq!(scrub(&body).text, "日本 \u{a0}é <redacted> → ünï");
|
||||
let seed = "é日 sk-aaaaaaaaaaaaaaaaaaaa -----BEGIN PRIVATE KEY----- authorization: é eyJa.b.c https://u:p@h/x";
|
||||
let chars: Vec<char> = seed.chars().collect();
|
||||
for start in 0..chars.len() {
|
||||
for end in start..=chars.len() {
|
||||
let _ = scrub(&chars[start..end].iter().collect::<String>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_distinctive_key_is_found_glued_to_other_text_but_the_generic_prefix_still_needs_a_word_start() {
|
||||
let ghp = format!("xxx{}", "ghp_".to_owned() + &"k2L9".repeat(9));
|
||||
assert_eq!(scrub(&ghp).text, "xxx<redacted>");
|
||||
assert_eq!(scrub("idAKIA0123456789ABCDEFend").hits, [ScrubKind::KeyShape]);
|
||||
assert_eq!(scrub(&format!("zzz{}", "xoxb-123456789012-aBcDeFgHiJkLmNoP")).hits, [ScrubKind::KeyShape]);
|
||||
assert_eq!(scrub(&format!("q{}", "AIza".to_owned() + &"Sy1_".repeat(9))).hits, [ScrubKind::KeyShape]);
|
||||
assert!(scrub("risk-averse-thinking-process task-force-alpha-bravo").hits.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_width_characters_inside_a_token_leave_no_tail_and_emoji_joiners_survive_when_nothing_matched() {
|
||||
let split = format!("sk-{}\u{200b}{}", "q7W3".repeat(4), "q7W3".repeat(4));
|
||||
let out = scrub(&format!("key {split} end"));
|
||||
assert_eq!(out.text, "key <redacted> end");
|
||||
let jwt = "eyJhbGciOiJIUzI1NiJ9.eyJz\u{feff}dWIiOiJmYWtlIn0.c2lnbmF0dXJl\u{200d}ZmFrZQ";
|
||||
assert_eq!(scrub(jwt).text, "<redacted>");
|
||||
let family = "family \u{1f468}\u{200d}\u{1f469}\u{200d}\u{1f467} here";
|
||||
assert_eq!(scrub(family).text, family);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matrix_tokens_and_query_values_are_found_and_prose_about_them_is_not() {
|
||||
for same in ["the access_token is a thing", "access_token: short", "syt_ is the prefix", "mct_ tokens", "bearer:short"] {
|
||||
assert_eq!(scrub(same).text, same, "{same}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1964,7 +1964,7 @@ fn refused_note(why: Failure) -> String {
|
|||
/// Every string of `value` through the one URL-credential redaction, so JSON structure is never inside a candidate run.
|
||||
fn redact_strings(value: &mut Value) {
|
||||
match value {
|
||||
Value::String(text) => *text = crate::self_run::redact_url_userinfo(text),
|
||||
Value::String(text) => *text = huginn_mind::redact::redact_url_userinfo(text),
|
||||
Value::Array(items) => items.iter_mut().for_each(redact_strings),
|
||||
Value::Object(map) => map.values_mut().for_each(redact_strings),
|
||||
_ => {}
|
||||
|
|
|
|||
|
|
@ -924,109 +924,9 @@ fn result_head(event: &serde_json::Map<String, Value>) -> String {
|
|||
head
|
||||
}
|
||||
|
||||
/// A result's words as they leave the actuator, for the journal and for her mind (the reply): redacted by `redact_url_userinfo`.
|
||||
/// A result's words as they leave the actuator, for the journal and for her mind (the reply): redacted by `huginn_mind::redact::redact_url_userinfo`.
|
||||
fn result_text(event: &serde_json::Map<String, Value>) -> String {
|
||||
redact_url_userinfo(event.get("result").and_then(Value::as_str).unwrap_or(""))
|
||||
}
|
||||
|
||||
/// The one redaction of URL credentials. Call it at each place a string the CLI produced enters the journal, her mind or chat: today
|
||||
/// the Self result (`result_text`: journal line and Reply) and the structured output of a Persona or Sleep result
|
||||
/// (`persona::structured_output`, string by string). It does not cover claude's stderr (inherited) or the CLI's transcript files.
|
||||
///
|
||||
/// A forge token rides in a remote URL, so every `scheme://` and every protocol-relative `//` opens an authority, read as git, curl and
|
||||
/// RFC 3986 read it: it ends at the first `/`, `?`, `#`, backslash, quote, backtick, parenthesis, bracket, angle bracket or ASCII whitespace
|
||||
/// (`is_authority_end`). Its userinfo is everything before the authority's LAST `@` (a raw `@` in a password still redacts), and non-ASCII
|
||||
/// characters, NBSP and U+2028 included, stay inside it. The userinfo becomes `<redacted>`: `scheme://<redacted>@host`. With a colon it is
|
||||
/// always redacted; without one it is a bare token on http, https, ftp, ftps, ws, wss, any `+http`/`+https` scheme (`git+https`), protocol-relative `//` and an unparseable scheme, and a plain
|
||||
/// username on any other scheme (`ssh://git@host`), which stays. A `@` after the authority (`github.com/@user`, `pkg@1.2.3`) is path, not
|
||||
/// userinfo, and is left alone. Scheme-less credentials are covered only in the shape `word:secret@host.tld/` at the start of an
|
||||
/// ASCII-whitespace-delimited word. Not covered: a password holding `/ ? # ` a quote or ASCII whitespace (no parser reads it as userinfo),
|
||||
/// scp-style `user:secret@host:path`, tokens in query strings, claude's stderr, the kept transcripts. Syntax only; no reading of meaning.
|
||||
pub(crate) fn redact_url_userinfo(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut rest = text;
|
||||
while let Some(at) = rest.find("//") {
|
||||
let (before, after) = rest.split_at(at);
|
||||
let body = &after[2..];
|
||||
out.push_str(before);
|
||||
out.push_str("//");
|
||||
// The scheme is the ASCII run before `:`. Without a colon, `//` opens an authority only at a word start (`a//b@c` is not one).
|
||||
let scheme = match before.strip_suffix(':') {
|
||||
// Only ASCII bytes match, so the byte after the last non-match is a character boundary.
|
||||
Some(head) => head[head.bytes().rposition(|c| !(c.is_ascii_alphanumeric() || matches!(c, b'+' | b'-' | b'.'))).map_or(0, |i| i + 1)..].to_ascii_lowercase(),
|
||||
None if before.chars().next_back().is_none_or(|c| is_break(c) || matches!(c, '"' | '\'' | '`' | '(' | '[' | '{' | '<' | '=' | ',')) => String::new(),
|
||||
None => {
|
||||
rest = body;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let end = body.find(is_authority_end).unwrap_or(body.len());
|
||||
let authority = &body[..end];
|
||||
match authority.rfind('@') {
|
||||
Some(i) if i > 0 && (authority[..i].contains(':') || carries_bare_token(&scheme)) => {
|
||||
out.push_str("<redacted>");
|
||||
out.push_str(&authority[i..]);
|
||||
}
|
||||
_ => out.push_str(authority),
|
||||
}
|
||||
rest = &body[end..];
|
||||
}
|
||||
out.push_str(rest);
|
||||
redact_schemeless(&out)
|
||||
}
|
||||
|
||||
/// Schemes whose lone userinfo is a token, not a username: web and ftp schemes, `ws`/`wss`, and `x+http`/`x+https` wrappers; `scheme` is lowercase.
|
||||
fn carries_bare_token(scheme: &str) -> bool {
|
||||
matches!(scheme, "" | "http" | "https" | "ftp" | "ftps" | "ws" | "wss") || scheme.ends_with("+http") || scheme.ends_with("+https")
|
||||
}
|
||||
|
||||
/// ASCII whitespace; NBSP, U+2028 and other Unicode whitespace are not, so a secret holding them is redacted whole.
|
||||
fn is_break(c: char) -> bool {
|
||||
matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c')
|
||||
}
|
||||
|
||||
/// The end of a URL's authority.
|
||||
fn is_authority_end(c: char) -> bool {
|
||||
is_break(c) || matches!(c, '/' | '?' | '#' | '\\' | '"' | '\'' | '`' | '(' | ')' | '[' | ']' | '{' | '}' | '<' | '>')
|
||||
}
|
||||
|
||||
/// `word:secret@host.tld/` at the start of a whitespace-delimited word that holds no `//` becomes `<redacted>@host.tld/`.
|
||||
fn redact_schemeless(text: &str) -> String {
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut word = String::new();
|
||||
let flush = |word: &mut String, out: &mut String| {
|
||||
out.push_str(&redact_word(word));
|
||||
word.clear();
|
||||
};
|
||||
for c in text.chars() {
|
||||
if is_break(c) {
|
||||
flush(&mut word, &mut out);
|
||||
out.push(c);
|
||||
} else {
|
||||
word.push(c);
|
||||
}
|
||||
}
|
||||
flush(&mut word, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn redact_word(word: &str) -> String {
|
||||
let shaped = (|| {
|
||||
if word.contains("//") {
|
||||
return None;
|
||||
}
|
||||
let at = word.rfind('@')?;
|
||||
let colon = word.find(':')?;
|
||||
let (user, host) = (&word[..colon], &word[at + 1..]);
|
||||
let host_name = &host[..host.find('/')?];
|
||||
let user_ok = !user.is_empty() && user.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'));
|
||||
let host_ok = host_name.contains('.') && host_name.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-'));
|
||||
(colon + 1 < at && user_ok && host_ok).then_some(at)
|
||||
})();
|
||||
match shaped {
|
||||
Some(at) => format!("<redacted>{}", &word[at..]),
|
||||
None => word.to_string(),
|
||||
}
|
||||
huginn_mind::redact::redact_url_userinfo(event.get("result").and_then(Value::as_str).unwrap_or(""))
|
||||
}
|
||||
|
||||
/// `text` in consecutive pieces of at most `limit` bytes, each cut at a character boundary; their concatenation is `text`.
|
||||
|
|
|
|||
|
|
@ -122,18 +122,6 @@ fn a_refused_piece_of_a_long_result_holds_back_the_pieces_behind_it() {
|
|||
// A canary stands where a forge token would: no byte of it may reach the journal or her mind.
|
||||
const CANARY: &str = "CANARY-NOT-A-REAL-TOKEN-0000";
|
||||
|
||||
#[test]
|
||||
fn url_userinfo_is_redacted_where_it_is_a_secret_and_kept_where_it_is_a_username() {
|
||||
let said = format!(
|
||||
"a https://user:{CANARY}@host.example/r.git b https://{CANARY}@host.example/r.git c ssh://u:{CANARY}@h/x d http://{CANARY}@h e HTTPS://x:{CANARY}@h \
|
||||
f https://u:p@ss{CANARY}@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://u:{CANARY}@h) k ://@x"
|
||||
);
|
||||
let want = "a https://<redacted>@host.example/r.git b https://<redacted>@host.example/r.git c ssh://<redacted>@h/x d http://<redacted>@h e HTTPS://<redacted>@h \
|
||||
f https://<redacted>@h/x g ssh://git@host.example/r.git h git://git@h/x i https://host.example/no-userinfo j (https://<redacted>@h) k ://@x";
|
||||
assert_eq!(redact_url_userinfo(&said), want);
|
||||
assert!(!redact_url_userinfo(&said).contains(CANARY));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_credential_in_a_url_reaches_neither_the_journal_nor_the_admitted_reply() {
|
||||
let mind = live_mind();
|
||||
|
|
@ -157,66 +145,6 @@ fn a_credential_split_by_the_reply_limit_is_redacted_before_the_cut() {
|
|||
assert!(!first.text.0.contains("CANARY") && !reply_of(&mind, 2).unwrap().text.0.contains("CANARY"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_secret_is_redacted_whole_by_the_authority_rule_and_commonplace_urls_are_left_alone() {
|
||||
let r = redact_url_userinfo;
|
||||
// Raw @, percent-encoding and non-ASCII (NBSP, U+2028) stay inside userinfo.
|
||||
for secret in ["pa@ssT", "pa%40ssT", "pa\u{a0}ssT", "pa\u{2028}ssT", "pa:ssT", "\u{e9}\u{65e5}ssT"] {
|
||||
let out = r(&format!("x https://u:{secret}@h/x"));
|
||||
assert_eq!(out, "x https://<redacted>@h/x", "{secret:?}");
|
||||
}
|
||||
// The authority ends at / ? # \\ quotes, brackets and ASCII whitespace: no parser reads those as userinfo, so what follows is path.
|
||||
assert_eq!(r("https://u:pa/ssT@h/x"), "https://u:pa/ssT@h/x");
|
||||
assert_eq!(r("https://u:pa ssT@h/x"), "https://u:pa ssT@h/x");
|
||||
assert_eq!(r("https://u:pa?ssT@h"), "https://u:pa?ssT@h");
|
||||
// Ordinary URLs with @ in the path are byte-identical.
|
||||
for same in [
|
||||
"https://github.com/@user/repo",
|
||||
"https://medium.com/@name/post-1",
|
||||
"https://registry.npmjs.org/@scope/pkg",
|
||||
"https://h/pkg@1.2.3",
|
||||
"a//b@c",
|
||||
"see //cdn.h/@x/y and https://h/?mail=a@b.c#f@g",
|
||||
"https://h:8080/p git@github.com:org/r.git ssh://git@host/r.git ssh://git@h:22/r git://git@h/x",
|
||||
] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
// Compact JSON: only the credentialed URL changes.
|
||||
let json = format!(r#"{{"a":"https://u:{CANARY}@h/x","b":"https://h2/@v","c":"https://h3/p@1"}}"#);
|
||||
assert_eq!(r(&json), r#"{"a":"https://<redacted>@h/x","b":"https://h2/@v","c":"https://h3/p@1"}"#);
|
||||
assert_eq!(r("a/https://u:T@h/x,https://v:T@h2"), "a/https://<redacted>@h/x,https://<redacted>@h2");
|
||||
assert_eq!(r("HTTPS://TOKEN@h"), "HTTPS://<redacted>@h");
|
||||
assert_eq!(r("[l](https://T@h/x) <https://T@h> `https://u:T@h`"), "[l](https://<redacted>@h/x) <https://<redacted>@h> `https://<redacted>@h`");
|
||||
assert_eq!(r("a //u:T@host/x b //T@host c \"//T@h\" (//T@h)"), "a //<redacted>@host/x b //<redacted>@host c \"//<redacted>@h\" (//<redacted>@h)");
|
||||
// Token-bearing schemes beyond http: +http/+https wrappers and ws/wss redact a bare token; git+ssh and ssh keep their username.
|
||||
assert_eq!(r("git+https://T@github.com/a/b GIT+HTTPS://T@h wss://T@h/ ws://T@h/"), "git+https://<redacted>@github.com/a/b GIT+HTTPS://<redacted>@h wss://<redacted>@h/ ws://<redacted>@h/");
|
||||
for same in ["git+ssh://git@host/x", "ssh://git@host/x"] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
// Scheme-less: only word:secret@host.tld/ . Each guard of the shape has its own case.
|
||||
assert_eq!(r("user:T@host.tld/path"), "<redacted>@host.tld/path");
|
||||
for same in ["a:b@c", "a:b@host/x", "a:b@ho_st.tld/x", "u$er:T@host.tld/x", "a:@host.tld/x", "mail me@x.org/ and :T@host.tld/x"] {
|
||||
assert_eq!(r(same), same);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multibyte_text_never_panics_the_redaction() {
|
||||
let r = redact_url_userinfo;
|
||||
for case in ["see \u{201c}https://h/x", "\u{2192}https://h/x", "docs\u{ff1a}https://h/x", "\u{e9}://x", "\u{65e5}\u{672c}://u:T@h", "https://x/\u{e9}://", "\u{e9}//u:T@h", "\u{e9}:u:T@h.tld/"] {
|
||||
let _ = r(case);
|
||||
}
|
||||
assert_eq!(r("\u{65e5}\u{672c}://u:T@h"), "\u{65e5}\u{672c}://<redacted>@h");
|
||||
assert_eq!(r("\u{2192}https://u:T@h/x"), "\u{2192}https://<redacted>@h/x");
|
||||
let seed = "\u{201c}\u{2192}\u{ff1a}\u{e9}\u{65e5}\u{a0}https://u:T@h/x//y:z@a.b/ ://@ //";
|
||||
let chars: Vec<char> = seed.chars().collect();
|
||||
for start in 0..=chars.len() {
|
||||
for end in start..=chars.len() {
|
||||
let _ = r(&chars[start..end].iter().collect::<String>());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persona_and_sleep_structured_output_is_redacted_string_by_string() {
|
||||
let line = json!({"type": "result", "structured_output": {"say": format!("https://u:{CANARY}@h/x,"), "n": 3, "list": [format!("https://{CANARY}@h")]}}).to_string();
|
||||
|
|
|
|||
|
|
@ -169,3 +169,32 @@ the cited message and its neighbours in the same room from this index, no
|
|||
embedder and no Synapse; an id the index lacks is refused naming `event_id`.
|
||||
The listing limit above is not changed by it: the cut spec does not assign that
|
||||
fix, so it stays open.
|
||||
|
||||
## Status: ingest core (cut chat-ingest-core)
|
||||
|
||||
`sources::chat::refresh_chat` writes chat points; scope is `sources/room_scope.rs`
|
||||
and the credential scrub is `huginn_mind::redact`. Rules the code holds, which a
|
||||
reader of the index must know:
|
||||
|
||||
- A text edit changes the stored body in place, only when its sender is the
|
||||
original's sender. Edits are never stored as events, so no payload field
|
||||
records which edit changed a body.
|
||||
- **Rebuild on an unknown redaction.** A redaction whose target is neither a
|
||||
message in the room's stored chunks nor an event of the same page may be of an
|
||||
edit whose own id Huginn never stored, so the body that edit set cannot be
|
||||
taken back from the chunk. The room is then rebuilt: its points and cursor are
|
||||
purged and the next read re-ingests its history from the port. Synapse serves a
|
||||
redacted edit without its content (verified against Synapse 1.164.1), so the
|
||||
original body returns and the edited text does not. Within one page, a redacted
|
||||
edit is reverted in the fold without a rebuild. Redactions the rebuild itself
|
||||
replays do not trigger another rebuild. A redaction of a media or state event
|
||||
also lands here (the cost is one rebuild of that room).
|
||||
- **Edits of a redacted original are dropped.** Synapse keeps serving the
|
||||
edits of a redacted original, with their replacement text, so Huginn drops any
|
||||
edit whose target is not a message it holds or has just read (a redacted,
|
||||
purged, never-stored or unknown original), live or late, in the fold and in a
|
||||
rebuild alike.
|
||||
- How the Matrix port maps a pruned event (`content: {}`) to `EventKind::Other`
|
||||
is the ingest-job cut's to pin; this core only assumes it does.
|
||||
- An error reading a joined room's facts is not a reclassification: the room
|
||||
keeps its points and cursor, and the run's report for the room says why.
|
||||
|
|
|
|||
Loading…
Reference in a new issue