diff --git a/crates/huginn-daemon/Cargo.toml b/crates/huginn-daemon/Cargo.toml index 93ffc47..15ada83 100644 --- a/crates/huginn-daemon/Cargo.toml +++ b/crates/huginn-daemon/Cargo.toml @@ -18,7 +18,7 @@ name = "huginn-daemon" path = "src/main.rs" # The source index's one writer: a oneshot job that brings `huginn_sources` to -# the forge's current state and exits. No daemon calls any of its writes. +# the forge's and the chat reader's current state and exits. No daemon calls any of its writes. [[bin]] name = "huginn-sources" path = "src/bin/huginn-sources.rs" @@ -34,7 +34,8 @@ rmp-serde = "1" rustix = { version = "1", default-features = false, features = ["std", "net"] } serde = { version = "1", features = ["derive"] } # The vector store's and the embedder's HTTP APIs are the xenos boundary, and -# JSON exists in `index/ollama.rs` and `index/qdrant.rs` only. `ureq` 3 with +# JSON exists in `index/ollama.rs`, `index/qdrant.rs`, `sources/forge.rs` and +# `sources/matrix.rs` only. `ureq` 3 with # default features off is plain HTTP with no TLS (`rustls` and `gzip` are its # only defaults), which is all the private mesh needs. It stays below 3.4, # whose `base64` 0.23 would put a second `base64` in a tree that has one. diff --git a/crates/huginn-daemon/src/bin/huginn-sources.rs b/crates/huginn-daemon/src/bin/huginn-sources.rs index fc5a291..e75e608 100644 --- a/crates/huginn-daemon/src/bin/huginn-sources.rs +++ b/crates/huginn-daemon/src/bin/huginn-sources.rs @@ -1,21 +1,38 @@ -//! The source index's refresh job: one run brings the `huginn_sources` -//! collection to the forge's current state, journals one line per repo, and -//! exits nonzero if any repo failed or the collection could not be used. +//! The source index's refresh job: one run brings the forge corpus and, when +//! configured, the rooms the chat reader has joined (minus the estate and DM +//! exclusion) to current state, journals one line per repo and room, and exits +//! nonzero if either failed or the collection could not be used. The +//! `chat-purge` subcommand writes a steward's purge tombstone and nothing else. +//! +//! Exit status: 0 clean, 1 a run failed, 2 bad arguments, 3 another run holds +//! the lock. The Matrix token is read from a file (a systemd credential), +//! never from the environment or argv. -use std::collections::BTreeMap; -use std::path::PathBuf; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{File, OpenOptions, TryLockError}; +use std::path::{Path, PathBuf}; use std::process::ExitCode; use anyhow::{Context, Result, ensure}; +use chrono::{DateTime, Utc}; use huginn_daemon::index::ollama::pool; use huginn_daemon::index::qdrant::QdrantIndex; use huginn_daemon::sources::forge::HttpForge; -use huginn_daemon::sources::refresh; +use huginn_daemon::sources::job::{self, Chat, Purge}; +use huginn_daemon::sources::matrix::HttpMatrix; +use huginn_daemon::sources::room_scope::{ESTATE_ALIASES, Estate}; /// Where the forge answers when the flag is absent: the loopback address of the /// forge on the host that runs this job. const DEFAULT_FORGE_URL: &str = "http://127.0.0.1:3310"; +/// The chat flags, which come all together or not at all. +struct ChatOptions { + matrix_url: String, + token: String, + estate: Estate, +} + struct Options { state_root: PathBuf, qdrant_url: String, @@ -25,75 +42,310 @@ struct Options { reserve_urls: Vec, embedding_model: String, forge_url: String, + chat: Option, + lock_file: Option, } -fn parse(args: impl Iterator) -> Result { +enum Command { + Run(Options), + Purge { qdrant_url: String, lock_file: PathBuf, purge: Purge }, +} + +/// How a run ends other than by finishing. +enum Stop { + Usage(anyhow::Error), + Locked, +} + +/// The options named, each once, except those in `repeated`, which keep every value. +fn collect(args: impl Iterator, once: &[&str], repeated: &[&str]) -> Result<(BTreeMap, BTreeMap>)> { let mut args = args; - let mut values: BTreeMap = BTreeMap::new(); - let mut ollama_urls = Vec::new(); - let mut reserve_urls = Vec::new(); + let mut singles = BTreeMap::new(); + let mut many: BTreeMap> = BTreeMap::new(); while let Some(name) = args.next() { let name = name.strip_prefix("--").with_context(|| format!("expected --option, got {name:?}"))?; - ensure!( - matches!(name, "state-root" | "qdrant-url" | "ollama-url" | "reserve-ollama-url" | "embedding-model" | "forge-url"), - "unsupported huginn-sources option --{name}" - ); + ensure!(once.contains(&name) || repeated.contains(&name), "unsupported huginn-sources option --{name}"); let value = args.next().with_context(|| format!("missing value for --{name}"))?; - if name == "ollama-url" || name == "reserve-ollama-url" { - ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}"); - if name == "ollama-url" { ollama_urls.push(value) } else { reserve_urls.push(value) } + if repeated.contains(&name) { + many.entry(name.to_owned()).or_default().push(value); } else { - ensure!(values.insert(name.to_owned(), value).is_none(), "duplicate option --{name}"); + ensure!(singles.insert(name.to_owned(), value).is_none(), "duplicate option --{name}"); } } - let take = |name: &str| values.get(name).cloned().with_context(|| format!("--{name} is required")); - let url = |name: &str, value: String| { - ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}"); - Ok(value) - }; - let state_root = PathBuf::from(take("state-root")?); - ensure!(state_root.is_absolute(), "--state-root must be an absolute path"); - let embedding_model = take("embedding-model")?; - ensure!(!embedding_model.is_empty(), "--embedding-model must name a model"); - Ok(Options { - state_root, - qdrant_url: url("qdrant-url", take("qdrant-url")?)?, - ollama_urls: { - ensure!(!ollama_urls.is_empty(), "at least one --ollama-url is required; repeat it for each GPU host"); - ollama_urls + Ok((singles, many)) +} + +fn http(name: &str, value: String) -> Result { + ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}"); + Ok(value) +} + +/// The Matrix reader's bearer travels in clear, and this build speaks no TLS: only a loopback host may be named. +fn loopback_http(name: &str, value: String) -> Result { + let value = http(name, value)?; + let authority = value["http://".len()..].split(['/', '?', '#']).next().unwrap_or(""); + let host = match authority.strip_prefix('[') { + // The bracket must end the authority or be followed by the port, so `[::1].evil.example` names no loopback. + Some(inner) => match inner.split_once(']') { + Some((host, "")) => host, + Some((host, port)) if port.starts_with(':') => host, + _ => "", }, + None => authority.split(':').next().unwrap_or(""), + }; + let loopback = !authority.contains('@') && (host.eq_ignore_ascii_case("localhost") || host.parse::().is_ok_and(|ip| ip.is_loopback())); + ensure!(loopback, "--{name} must name a loopback host (localhost, 127.0.0.0/8 or [::1]): the bearer token travels in clear and this build has no TLS, got {value:?}"); + Ok(value) +} + +fn absolute(name: &str, value: &str) -> Result { + let path = PathBuf::from(value); + ensure!(path.is_absolute(), "--{name} must be an absolute path"); + Ok(path) +} + +fn chat_options(singles: &BTreeMap, excluded: Vec) -> Result> { + let given = [singles.contains_key("matrix-url"), singles.contains_key("matrix-token-file"), !excluded.is_empty()]; + if !given.contains(&true) { + return Ok(None); + } + for (present, name) in given.iter().zip(["matrix-url", "matrix-token-file", "chat-exclude-room"]) { + ensure!(*present, "--{name} is missing: the chat flags (--matrix-url, --matrix-token-file, --chat-exclude-room) come all together or not at all"); + } + let ids: BTreeSet = excluded.iter().cloned().collect(); + ensure!(ids.len() == excluded.len(), "--chat-exclude-room must not repeat a room id"); + ensure!(ids.len() == ESTATE_ALIASES.len(), "--chat-exclude-room must be given exactly {} times (the ids of {})", ESTATE_ALIASES.len(), ESTATE_ALIASES.join(", ")); + ensure!(ids.iter().all(|id| id.starts_with('!')), "--chat-exclude-room takes room ids beginning with !, not aliases"); + let path = absolute("matrix-token-file", &singles["matrix-token-file"])?; + let token = std::fs::read_to_string(&path).with_context(|| format!("reading --matrix-token-file {}", path.display()))?.trim().to_owned(); + ensure!(!token.is_empty(), "--matrix-token-file {} is empty", path.display()); + Ok(Some(ChatOptions { matrix_url: loopback_http("matrix-url", singles["matrix-url"].clone())?, token, estate: Estate::new(ids)? })) +} + +fn parse(mut args: impl Iterator, now: DateTime) -> Result { + let mut rest: Vec = Vec::new(); + let first = args.next(); + if first.as_deref() == Some("chat-purge") { + let (singles, _) = collect(args, &["qdrant-url", "room", "from", "to", "request", "lock-file"], &[])?; + let take = |name: &str| singles.get(name).cloned().with_context(|| format!("--{name} is required")); + return Ok(Command::Purge { + qdrant_url: http("qdrant-url", take("qdrant-url")?)?, + lock_file: absolute("lock-file", &take("lock-file")?)?, + purge: Purge::check(&take("room")?, &take("from")?, &take("to")?, &take("request")?, now)?, + }); + } + rest.extend(first); + let (mut singles, mut many) = collect( + rest.into_iter().chain(args), + &["state-root", "qdrant-url", "embedding-model", "forge-url", "matrix-url", "matrix-token-file", "lock-file"], + &["ollama-url", "reserve-ollama-url", "chat-exclude-room"], + )?; + let mut hosts = |name: &str| -> Result> { many.remove(name).unwrap_or_default().into_iter().map(|url| http(name, url)).collect() }; + let ollama_urls = hosts("ollama-url")?; + let reserve_urls = hosts("reserve-ollama-url")?; + let excluded = many.remove("chat-exclude-room").unwrap_or_default(); + let take = |singles: &mut BTreeMap, name: &str| singles.remove(name).with_context(|| format!("--{name} is required")); + let state_root = absolute("state-root", &take(&mut singles, "state-root")?)?; + let embedding_model = take(&mut singles, "embedding-model")?; + ensure!(!embedding_model.is_empty(), "--embedding-model must name a model"); + ensure!(!ollama_urls.is_empty(), "at least one --ollama-url is required; repeat it for each GPU host"); + let chat = chat_options(&singles, excluded)?; + let lock_file = singles.get("lock-file").map(|path| absolute("lock-file", path)).transpose()?; + ensure!(chat.is_none() || lock_file.is_some(), "--lock-file is required whenever the chat flags are given"); + Ok(Command::Run(Options { + state_root, + qdrant_url: http("qdrant-url", take(&mut singles, "qdrant-url")?)?, + ollama_urls, reserve_urls, embedding_model, - forge_url: url("forge-url", values.get("forge-url").cloned().unwrap_or_else(|| DEFAULT_FORGE_URL.to_owned()))?, - }) + forge_url: http("forge-url", singles.remove("forge-url").unwrap_or_else(|| DEFAULT_FORGE_URL.to_owned()))?, + chat, + lock_file, + })) +} + +/// Takes the job's lock for the life of the returned file. A held lock means a run (or a +/// purge request) is in progress and nothing may change: the cursors object is replaced whole. +fn take_lock(path: &Path) -> Result { + let file = OpenOptions::new().create(true).truncate(false).write(true).open(path).with_context(|| format!("opening --lock-file {}", path.display())).map_err(Stop::Usage)?; + match file.try_lock() { + Ok(()) => Ok(file), + Err(TryLockError::WouldBlock) => Err(Stop::Locked), + Err(TryLockError::Error(error)) => Err(Stop::Usage(anyhow::Error::new(error).context(format!("locking --lock-file {}", path.display())))), + } +} + +fn lock_if(path: &Option) -> Result, Stop> { + path.as_deref().map(take_lock).transpose() +} + +fn execute(command: Command) -> Result { + match command { + Command::Purge { qdrant_url, lock_file, purge } => { + let _lock = take_lock(&lock_file)?; + let mut index = QdrantIndex::new(&qdrant_url); + Ok(match job::purge(&mut index, &purge) { + Ok(line) => { + println!("{line}"); + 0 + } + Err(error) => { + eprintln!("huginn-sources: {error:#}"); + 1 + } + }) + } + Command::Run(options) => { + let _lock = lock_if(&options.lock_file)?; + let mut embedder = pool(&options.ollama_urls, &options.reserve_urls, &options.embedding_model); + let mut index = QdrantIndex::new(&options.qdrant_url); + let mut forge = HttpForge::new(&options.forge_url); + let mut matrix = options.chat.as_ref().map(|chat| HttpMatrix::new(&chat.matrix_url, &chat.token)); + let chat = matrix.as_mut().zip(options.chat.as_ref()).map(|(history, chat)| Chat { history, estate: &chat.estate }); + Ok(job::run(&mut embedder, &mut index, &mut forge, &options.state_root, chat, &mut |line| println!("{line}"), &mut |line| eprintln!("{line}"))) + } + } } fn main() -> ExitCode { - let options = match parse(std::env::args().skip(1)) { - Ok(options) => options, - Err(error) => { + match parse(std::env::args().skip(1), Utc::now()).map_err(Stop::Usage).and_then(execute) { + Ok(status) => ExitCode::from(status), + Err(Stop::Usage(error)) => { eprintln!("huginn-sources: {error:#}"); - return ExitCode::from(2); + ExitCode::from(2) } - }; - let mut embedder = pool(&options.ollama_urls, &options.reserve_urls, &options.embedding_model); - let mut index = QdrantIndex::new(&options.qdrant_url); - let mut forge = HttpForge::new(&options.forge_url); - let mut journal = |report: &huginn_daemon::sources::Report| println!("{report}"); - match refresh(&mut embedder, &mut index, &mut forge, &options.state_root, &mut journal) { - Ok(refreshed) if refreshed.is_clean() => ExitCode::SUCCESS, - Ok(refreshed) => { - if !refreshed.failed.is_empty() { - eprintln!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", ")); - } - if let Some(why) = &refreshed.retirement_withheld { - eprintln!("huginn-sources: no repo was retired: {why}"); - } - ExitCode::from(1) - } - Err(error) => { - eprintln!("huginn-sources: {error:#}"); - ExitCode::from(1) + Err(Stop::Locked) => { + eprintln!("huginn-sources: huginn-sources is running"); + ExitCode::from(3) } } } + +#[cfg(test)] +mod tests { + use super::*; + + const NOW: &str = "2026-11-17T00:00:00Z"; + + fn args(text: &str) -> impl Iterator { + text.split_whitespace().map(str::to_owned).collect::>().into_iter() + } + + fn now() -> DateTime { + DateTime::parse_from_rfc3339(NOW).unwrap().with_timezone(&Utc) + } + + fn error(text: &str) -> String { + match parse(args(text), now()) { + Ok(_) => panic!("parsed: {text}"), + Err(error) => format!("{error:#}"), + } + } + + const BASE: &str = "--state-root /s --qdrant-url http://q --ollama-url http://o --embedding-model m"; + const IDS: &str = "--chat-exclude-room !a:s --chat-exclude-room !b:s --chat-exclude-room !c:s"; + + fn with_token(secret: &str) -> (tempfile::TempDir, String) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("token"); + std::fs::write(&path, secret).unwrap(); + let path = path.display().to_string(); + (dir, path) + } + + #[test] + fn chat_flags_come_all_or_none() { + let (_dir, token) = with_token(" sekret-token\n"); + let lock = "--lock-file /l"; + let full = format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS} {lock}"); + let Ok(Command::Run(options)) = parse(args(&full), now()) else { panic!("full chat flags must parse") }; + assert_eq!(options.chat.as_ref().unwrap().token, "sekret-token", "trimmed"); + assert!(matches!(parse(args(BASE), now()), Ok(Command::Run(Options { chat: None, lock_file: None, .. })))); + assert!(error(&format!("{BASE} --matrix-url http://localhost {lock}")).contains("--matrix-token-file is missing")); + assert!(error(&format!("{BASE} --matrix-token-file {token} {IDS} {lock}")).contains("--matrix-url is missing")); + assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {lock}")).contains("--chat-exclude-room is missing")); + assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS}")).contains("--lock-file is required")); + } + + #[test] + fn chat_flag_values_are_checked_and_never_print_the_token() { + let (_dir, token) = with_token("sekret-token"); + let on = |extra: &str| format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} --lock-file /l {extra}"); + assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !b:s")).contains("exactly 3")); + assert!(error(&on(&format!("{IDS} --chat-exclude-room !d:s"))).contains("exactly 3")); + assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !a:s --chat-exclude-room !b:s")).contains("repeat")); + assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !b:s --chat-exclude-room #c:s")).contains("beginning with !")); + assert!(error(&format!("{BASE} --matrix-url https://m --matrix-token-file {token} --lock-file /l {IDS}")).contains("plain http://")); + assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file rel --lock-file /l {IDS}")).contains("absolute")); + assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} --lock-file rel {IDS}")).contains("absolute")); + let (_empty, empty) = with_token(" \n"); + let message = error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {empty} --lock-file /l {IDS}")); + assert!(message.contains(&empty) && message.contains("empty")); + let missing = error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file /nowhere/t --lock-file /l {IDS}")); + assert!(missing.contains("/nowhere/t")); + for text in [on(IDS), on("--chat-exclude-room !a:s")] { + assert!(!error(&format!("{text} --nope x")).contains("sekret-token")); + } + let dead = ["--chat", "room"].join("-"); + assert!(error(&format!("{BASE} {dead} !x:s")).contains(&format!("unsupported huginn-sources option {dead}"))); + } + + fn purge(extra: &str) -> String { + error(&format!("chat-purge --qdrant-url http://q --lock-file /l {extra}")) + } + + #[test] + fn chat_purge_arguments() { + let good = "--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z --request ref-1"; + let Ok(Command::Purge { purge: asked, .. }) = parse(args(&format!("chat-purge --qdrant-url http://q --lock-file /l {good}")), now()) else { panic!() }; + assert_eq!((asked.window.from_ms, asked.window.to_ms), (1_794_268_800_000, 1_794_355_200_000)); + let bad = |from: &str, to: &str, room: &str, request: &str| purge(&format!("--room {room} --from {from} --to {to} --request {request}")); + let (f, t) = ("2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z"); + assert!(bad(f, t, "#a:s", "r").contains("--room")); + assert!(bad("2026-11-10T00:00:00+00:00", t, "!r:s", "r").contains("--from")); + assert!(bad(f, "2026-11-11T00:00:00", "!r:s", "r").contains("--to")); + assert!(bad(t, f, "!r:s", "r").contains("before")); + assert!(bad(f, "2026-11-18T00:00:00Z", "!r:s", "r").contains("after now")); + assert!(bad("2025-11-09T00:00:00Z", t, "!r:s", "r").contains("366")); + assert!(purge(&format!("--room !r:s --from {f} --to {t} --request {}", "x".repeat(121))).contains("--request")); + assert!(purge("--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z").contains("--request is required")); + assert!(parse(args(&format!("chat-purge --qdrant-url http://q --lock-file /l {good} --matrix-url http://localhost")), now()).is_err(), "no Matrix flags"); + } + + #[test] + fn matrix_url_must_be_loopback_http() { + let (_dir, token) = with_token("sekret-token"); + let with = |url: &str| format!("{BASE} --matrix-url {url} --matrix-token-file {token} --lock-file /l {IDS}"); + for good in ["http://localhost", "http://127.0.0.1:8008", "http://127.9.9.9/", "http://[::1]:8008", "http://LOCALHOST:1/x"] { + assert!(parse(args(&with(good)), now()).is_ok(), "{good}"); + } + for bad in ["http://matrix.example", "http://10.0.0.5:8008", "http://localhost@evil.example", "http://127.0.0.1.evil.example", "http://localhost.evil.example", "http://[::2]", "https://localhost", "http://127.0.0.1:pw@evil.example", "http://[::1].evil.example", "http://[::1]x"] { + assert!(error(&with(bad)).contains("--matrix-url"), "{bad}"); + } + } + + #[test] + fn a_held_lock_exits_3_through_execute_for_both_modes() { + let dir = tempfile::tempdir().unwrap(); + let lock = dir.path().join("lock"); + let held = take_lock(&lock).ok().unwrap(); + let lock = lock.display().to_string(); + let good = "--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z --request r"; + let purge = parse(args(&format!("chat-purge --qdrant-url http://127.0.0.1:1 --lock-file {lock} {good}")), now()).unwrap(); + assert!(matches!(execute(purge), Err(Stop::Locked))); + let (_t, token) = with_token("sekret-token"); + let run = parse(args(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS} --lock-file {lock}")), now()).unwrap(); + assert!(matches!(execute(run), Err(Stop::Locked))); + drop(held); + } + + #[test] + fn a_held_lock_stops_the_run_untouched() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("lock"); + let held = take_lock(&path).ok().unwrap(); + assert!(matches!(take_lock(&path), Err(Stop::Locked))); + drop(held); + assert!(take_lock(&path).is_ok()); + } +} diff --git a/crates/huginn-daemon/src/sources/chat.rs b/crates/huginn-daemon/src/sources/chat.rs index fcee66f..808ed83 100644 --- a/crates/huginn-daemon/src/sources/chat.rs +++ b/crates/huginn-daemon/src/sources/chat.rs @@ -134,6 +134,39 @@ impl std::fmt::Display for ChatReport { } } +/// One line of a chat run's journal, typed so the job routes by what happened and not by how the line reads. +#[derive(Clone, Debug, PartialEq)] +pub enum Entry { + /// A room's report; a failed one is an alarm. + Report(ChatReport), + Refused(RefusedRoom), + Scrubbed(ScrubHit), + /// A failure outside any one room's report: retiring rooms. + Trouble(String), +} + +impl Entry { + /// The operator must see this on stderr: a failure, a refusal, a scrub hit. + pub fn is_alarm(&self) -> bool { + match self { + Self::Report(report) => report.failure.is_some(), + Self::Refused(_) | Self::Scrubbed(_) | Self::Trouble(_) => true, + } + } +} + +impl std::fmt::Display for Entry { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Report(report) => report.fmt(formatter), + Self::Refused(RefusedRoom { room_id, reason, event_id: None }) => write!(formatter, "refused: {room_id} {reason}"), + Self::Refused(RefusedRoom { room_id, reason, event_id: Some(event) }) => write!(formatter, "refused: {room_id} {reason} [{event}]"), + Self::Scrubbed(hit) => write!(formatter, "scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind), + Self::Trouble(why) => write!(formatter, "huginn-sources: {why}"), + } + } +} + /// A joined room the scope refused, whole, and why; `event_id` when an event did it. #[derive(Clone, Debug, PartialEq, Eq)] pub struct RefusedRoom { @@ -562,7 +595,7 @@ pub fn refresh_chat( index: &mut S, history: &mut H, estate: &Estate, - journal: &mut dyn FnMut(String), + journal: &mut dyn FnMut(Entry), ) -> Result { let DescribedSources::Labelled { meta, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else { bail!("the source collection is absent or not Huginn's; the forge refresh creates it and chat never does"); @@ -589,13 +622,14 @@ pub fn refresh_chat( Ok(Err(reason)) => { // Not kept, so the retirement below purges whatever it held. keep.remove(room_id); - journal(format!("refused: {room_id} {reason}")); - out.refused.push(RefusedRoom { room_id: room_id.clone(), reason, event_id: None }); + let refused = RefusedRoom { room_id: room_id.clone(), reason, event_id: None }; + journal(Entry::Refused(refused.clone())); + out.refused.push(refused); } // An error is not a reclassification: the room keeps its points and cursor, and the report says why. Err(error) => { let report = ChatReport::failed(room_id, None, &error.context("reading the room's facts")); - journal(report.to_string()); + journal(Entry::Report(report.clone())); out.failed.push(room_id.clone()); out.rooms.push(report); } @@ -604,35 +638,35 @@ pub fn refresh_chat( match retire_chat(index, &keep) { Ok(done) => { for (room, why) in done.failed { - journal(format!("huginn-sources: retiring {room} failed, the others go on: {why}")); + journal(Entry::Trouble(format!("retiring {room} failed, the others go on: {why}"))); out.failed.push(room); } } Err(error) => { - journal(format!("huginn-sources: retiring rooms failed: {error:#}")); + journal(Entry::Trouble(format!("retiring rooms failed: {error:#}"))); out.failed.push("retire".to_owned()); } } for facts in admitted { match ingest_room(embedder, index, history, identity.dimensions, &facts, &mut out) { Ok(Outcome::Done(report)) => { - journal(report.to_string()); + journal(Entry::Report(report.clone())); out.rooms.push(report); } Ok(Outcome::Refused(refused)) => { - journal(format!("refused: {} {} [{}]", refused.room_id, refused.reason, refused.event_id.as_deref().unwrap_or(""))); + journal(Entry::Refused(refused.clone())); out.refused.push(refused); } Err(error) => { let report = ChatReport::failed(&facts.room_id, facts.alias.clone(), &error); - journal(report.to_string()); + journal(Entry::Report(report.clone())); out.failed.push(facts.room_id.clone()); out.rooms.push(report); } } } for hit in &out.scrubbed { - journal(format!("scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind)); + journal(Entry::Scrubbed(hit.clone())); } Ok(out) } diff --git a/crates/huginn-daemon/src/sources/chat_tests.rs b/crates/huginn-daemon/src/sources/chat_tests.rs index 838d3fe..1d42c33 100644 --- a/crates/huginn-daemon/src/sources/chat_tests.rs +++ b/crates/huginn-daemon/src/sources/chat_tests.rs @@ -141,7 +141,7 @@ impl Rig { fn run(&mut self, history: &mut FakeHistory) -> (Result, Vec) { let mut journal = Vec::new(); - let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line)); + let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line.to_string())); (out, journal) } diff --git a/crates/huginn-daemon/src/sources/forge.rs b/crates/huginn-daemon/src/sources/forge.rs index dd700db..8bb1335 100644 --- a/crates/huginn-daemon/src/sources/forge.rs +++ b/crates/huginn-daemon/src/sources/forge.rs @@ -322,6 +322,7 @@ mod tests { use serde_json::json; use super::*; + use crate::sources::test_http::{Reply, forge_of}; /// Only public, live, original, non-empty repos are the corpus; a repo the /// forge does not say is safe is not indexed. @@ -400,60 +401,6 @@ mod tests { assert!(HttpForge::new(&unstated).catalog().unwrap().shortfall.unwrap().contains("X-Total-Count")); } - /// What a fake forge answers: a status, the `Location` header when there is one, and a body. - struct Reply { - status: u16, - location: Option, - body: String, - } - - impl From for Reply { - fn from(body: String) -> Self { - Self { status: 200, location: None, body } - } - } - - impl Reply { - /// What the live forge answers an anonymous reader for a private repo. - fn not_found() -> Self { - Self { status: 404, location: None, body: json!({ "message": "The target couldn't be found." }).to_string() } - } - - /// What the live forge answers for a renamed repo: a 301 to its public address, `/forge` prefix and all. - fn moved_to(location: &str) -> Self { - Self { status: 301, location: Some(location.to_owned()), body: String::new() } - } - } - - /// Answers each request by its path from `routes`; a path it has no answer for is held open and never answered, as a hung forge - /// does. Every path asked is recorded in order. - fn forge_of>(routes: impl Fn(&str) -> Option + Send + 'static) -> (String, std::sync::Arc>>) { - use std::io::{Read, Write}; - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let base = format!("http://{}", listener.local_addr().unwrap()); - let asked = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let record = std::sync::Arc::clone(&asked); - std::thread::spawn(move || { - let mut held = Vec::new(); - for stream in listener.incoming() { - let mut stream = stream.unwrap(); - let mut buffer = [0_u8; 4096]; - let read = stream.read(&mut buffer).unwrap(); - let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); - let path = request.split_whitespace().nth(1).unwrap_or_default().to_owned(); - record.lock().unwrap().push(path.clone()); - match routes(&path).map(Into::into) { - Some(Reply { status, location, body }) => { - let location = location.map(|location| format!("location: {location}\r\n")).unwrap_or_default(); - write!(stream, "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\n{location}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap(); - } - None => held.push(stream), - } - } - }); - (base, asked) - } - const FEED_SAMPLE: &str = include_str!("../../tests/fixtures/feed.json"); const NOW: &str = "2026-11-03T12:00:00Z"; @@ -523,7 +470,7 @@ mod tests { assert!(read.open_prs.is_empty() && read.unseen.is_empty() && read.capped.is_empty() && read.moved.is_empty(), "a cursor asks for neither list"); let wire = serde_json::to_string(&read).unwrap(); assert!(!wire.contains("CANARY") && !wire.contains("obey") && !wire.contains("Message"), "no forge text leaves the read: {wire}"); - let asked = asked.lock().unwrap().clone(); + let asked = asked.paths(); assert!(!asked.is_empty() && asked.iter().all(|path| names.iter().any(|name| path.starts_with(&format!("/api/v1/repos/{name}/activities/feeds")))), "{asked:?}"); } @@ -555,13 +502,13 @@ mod tests { let far = HttpForge::new(&base).activity(&wanted, &me, Some(5), now).unwrap(); assert!(far.capped == repos(&["GameCult/Huginn"]), "stopped at the cap with the cursor not reached"); assert_eq!((far.events.len(), far.head), (200, 2500)); - assert_eq!(asked.lock().unwrap().len(), 4, "four pages and no more"); + assert_eq!(asked.paths().len(), 4, "four pages and no more"); let (base, asked) = forge_of(feed("nibu")); let near = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap(); assert!(near.capped.is_empty()); assert_eq!((near.events.len(), near.head), (20, 2500), "ids above the cursor only"); - assert_eq!(asked.lock().unwrap().len(), 1); + assert_eq!(asked.paths().len(), 1); let (base, _) = forge_of(feed("eureka")); let hers = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap(); @@ -583,7 +530,7 @@ mod tests { let read = HttpForge::new(&base).activity(&repos(&["GameCult/Ratatoskr"]), &me(), None, at(NOW)).unwrap(); assert_eq!(read.events.iter().map(|event| event.id).collect::>(), [300]); assert!(read.capped.is_empty()); - assert!(asked.lock().unwrap().iter().all(|path| !path.contains("/orgs/GameCult/activities"))); + assert!(asked.paths().iter().all(|path| !path.contains("/orgs/GameCult/activities"))); } /// A first read keeps what is inside 72 hours and nothing older, and its cursor is the newest id it saw, an entry past the window @@ -619,13 +566,13 @@ mod tests { let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) }); let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap(); assert_eq!((read.events.len(), read.capped.len()), (53, 0)); - assert_eq!(asked.lock().unwrap().iter().filter(|path| is_feed(path)).count(), 2, "page 2 was asked, page 3 was not"); + assert_eq!(asked.paths().iter().filter(|path| is_feed(path)).count(), 2, "page 2 was asked, page 3 was not"); let entries: Vec = (0..400).map(|n| push_at(5000 - n, "GameCult/Huginn", "Nibu", 5)).collect(); let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) }); let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap(); assert_eq!((read.events.len(), read.capped), (REPO_FEED_PAGES * PAGE, repos(&["GameCult/Huginn"])), "200 read and the rest disclosed"); - assert_eq!(asked.lock().unwrap().iter().filter(|path| is_feed(path)).count(), REPO_FEED_PAGES); + assert_eq!(asked.paths().iter().filter(|path| is_feed(path)).count(), REPO_FEED_PAGES); } /// With no cursor the read also lists the open PRs in her repos and hers anywhere, and the repos the forge's listing does not hold. @@ -660,14 +607,14 @@ mod tests { assert_eq!(first.unseen, repos(&["GameCult/Ghost"]), "r60 is on page 2 of the listing"); let window = at("2026-10-31T12:00:00Z").timestamp_millis() as u64; assert!(!first.events.is_empty() && first.events.iter().all(|event| event.at >= window), "inside 72 h: {:?}", first.events); - let paths = asked.lock().unwrap().clone(); + let paths = asked.paths(); let count = |what: &str| paths.iter().filter(|path| path.contains(what)).count(); assert_eq!((count("/issues/search"), count("/orgs/GameCult/repos")), (2, 2), "a short page ends each: {paths:?}"); // A cursor asks for neither list. let before = paths.len(); HttpForge::new(&base).activity(&wanted, &me(), Some(1), at(NOW)).unwrap(); - let after = asked.lock().unwrap().clone(); + let after = asked.paths(); assert!(after[before..].iter().all(|path| is_feed(path)), "only the feeds: {after:?}"); } @@ -693,12 +640,12 @@ mod tests { }); let read = HttpForge::new(&base).activity(&repos(&["GameCult/Huginn"]), &me(), None, at(NOW)).unwrap(); assert_eq!(read.open_prs.iter().map(|pr| pr.number).collect::>(), [7], "page 3 was read"); - assert_eq!(asked.lock().unwrap().iter().filter(|path| path.contains("/issues/search")).count(), 4, "to the short page"); + assert_eq!(asked.paths().iter().filter(|path| path.contains("/issues/search")).count(), 4, "to the short page"); } /// The forge as it answers an anonymous reader: public repos give their feed (`[]` when quiet), a private repo answers 404, a /// renamed repo answers 301 to a path under `/forge` that this loopback does not serve (asking it gives 404). - fn live_forge(entries: Vec, public: &'static [&'static str], listing: String) -> (String, std::sync::Arc>>) { + fn live_forge(entries: Vec, public: &'static [&'static str], listing: String) -> (String, crate::sources::test_http::Asked) { forge_of(move |path| { if path.starts_with("/forge/") { return Some(Reply::not_found()); @@ -733,7 +680,7 @@ mod tests { assert_eq!(first.unseen, repos(&["GameCult/gamecult-ops", "GameCult/cinny"])); assert_eq!(first.moved, [RepoMoved { from: "GameCult/cinny".into(), to: "GameCult/Ratatoskr".into() }]); assert!(first.capped.is_empty()); - assert!(asked.lock().unwrap().iter().all(|path| !path.starts_with("/forge/")), "a redirect is not followed: {:?}", asked.lock().unwrap()); + assert!(asked.paths().iter().all(|path| !path.starts_with("/forge/")), "a redirect is not followed: {:?}", asked.paths()); let later = forge.activity(&wanted, &me(), Some(250), at(NOW)).unwrap(); assert_eq!((later.events.iter().map(|event| event.id).collect::>(), later.head), (vec![300], 300), "a later read goes on past the dark repos"); diff --git a/crates/huginn-daemon/src/sources/job.rs b/crates/huginn-daemon/src/sources/job.rs new file mode 100644 index 0000000..dd08d2c --- /dev/null +++ b/crates/huginn-daemon/src/sources/job.rs @@ -0,0 +1,128 @@ +//! What one `huginn-sources` run does with the pieces it is handed: the forge +//! refresh, then the chat refresh (or, with chat off, the retirement of every +//! chat point), one exit status for both, and the steward-purge request. The +//! binary supplies the real forge, embedder, vector store and Matrix client and +//! owns the flags and the lock; this file decides nothing about scope or +//! scrubbing (`room_scope` and `redact` do) and never sees a credential. + +use std::collections::BTreeSet; +use std::path::Path; + +use anyhow::{Context, Result, bail, ensure}; +use chrono::{DateTime, Utc}; + +use super::chat::{Entry, RoomHistory, refresh_chat, retire_chat}; +use super::chat_purge::{PurgeWindow, parse_windows, purge_key, request_purge}; +use super::room_scope::Estate; +use super::{DescribedSources, Forge, SourceIndex, refresh}; +use crate::index::{Embedder, INDEX_CALL_TIMEOUT}; + +/// Chat on: the reader's rooms and the estate ids the scope refuses. +pub struct Chat<'a, H> { + pub history: &'a mut H, + pub estate: &'a Estate, +} + +/// One run: the forge refresh, then chat (`Some`) or the retirement of every chat point (`None`). +/// `say` takes the journal's report lines (stdout), `complain` the failures, refusals and scrub hits (stderr). +/// A forge failure never skips the chat run and a chat failure never hides a forge one. +/// Returns the exit status: 0 only when both were clean. +pub fn run( + embedder: &mut E, + index: &mut S, + forge: &mut F, + state_root: &Path, + chat: Option>, + say: &mut dyn FnMut(String), + complain: &mut dyn FnMut(String), +) -> u8 { + let mut clean = true; + match refresh(embedder, index, forge, state_root, &mut |report| say(report.to_string())) { + Ok(refreshed) if refreshed.is_clean() => {} + Ok(refreshed) => { + clean = false; + if !refreshed.failed.is_empty() { + complain(format!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", "))); + } + if let Some(why) = &refreshed.retirement_withheld { + complain(format!("huginn-sources: no repo was retired: {why}")); + } + } + Err(error) => { + clean = false; + complain(format!("huginn-sources: {error:#}")); + } + } + match chat { + Some(Chat { history, estate }) => { + let mut line = |entry: Entry| if entry.is_alarm() { complain(entry.to_string()) } else { say(entry.to_string()) }; + match refresh_chat(embedder, index, history, estate, &mut line) { + Ok(refreshed) => clean &= refreshed.is_clean(), + Err(error) => { + clean = false; + complain(format!("huginn-sources: chat: {error:#}")); + } + } + } + None => match retire_chat(index, &BTreeSet::new()) { + Ok(done) => { + if !done.retired.is_empty() { + say(format!("chat retired: {} rooms", done.retired.len())); + } + for (room, why) in done.failed { + clean = false; + complain(format!("huginn-sources: retiring {room} failed: {why}")); + } + } + Err(error) => { + clean = false; + complain(format!("huginn-sources: chat: {error:#}")); + } + }, + } + u8::from(!clean) +} + +/// The longest span one request may name. +const PURGE_SPAN_DAYS: i64 = 366; +const REQUEST_BYTES_MAX: usize = 120; + +/// A steward's purge request as the admin typed it, checked. Each error names its option. +#[derive(Debug, PartialEq, Eq)] +pub struct Purge { + pub room_id: String, + pub from: String, + pub to: String, + pub request: String, + pub window: PurgeWindow, +} + +fn instant(option: &str, text: &str) -> Result> { + ensure!(text.ends_with('Z'), "--{option} must be an RFC 3339 UTC time ending in Z, got {text:?}"); + let at = DateTime::parse_from_rfc3339(text).map_err(|_| anyhow::anyhow!("--{option} must be an RFC 3339 UTC time ending in Z, got {text:?}"))?; + Ok(at.with_timezone(&Utc)) +} + +impl Purge { + pub fn check(room: &str, from: &str, to: &str, request: &str, now: DateTime) -> Result { + ensure!(room.starts_with('!'), "--room must be a room id beginning with !, got {room:?}: an alias needs Matrix, and the journal prints both #alias (!id)"); + let (start, end) = (instant("from", from)?, instant("to", to)?); + ensure!(start < end, "--from must be before --to"); + ensure!(end <= now, "--to must not be after now"); + ensure!(end - start <= chrono::Duration::days(PURGE_SPAN_DAYS), "the window must be at most {PURGE_SPAN_DAYS} days"); + ensure!(!request.is_empty() && request.len() <= REQUEST_BYTES_MAX, "--request must be 1 to {REQUEST_BYTES_MAX} bytes"); + let ms = |at: DateTime| u64::try_from(at.timestamp_millis()).context("a purge time before 1970"); + Ok(Self { room_id: room.to_owned(), from: from.to_owned(), to: to.to_owned(), request: request.to_owned(), window: PurgeWindow { from_ms: ms(start)?, to_ms: ms(end)? } }) + } +} + +/// Writes the tombstone and returns the line the admin sees; the free-text request is quoted and escaped so it cannot forge a journal line. Touches no point and makes no Matrix call; +/// the caller holds the job's lock. The next run applies it. +pub fn purge(index: &mut S, purge: &Purge) -> Result { + request_purge(index, &purge.room_id, purge.window)?; + let DescribedSources::Labelled { cursors, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else { + bail!("the source collection is absent or not Huginn's"); + }; + let windows = cursors.get(&purge_key(&purge.room_id)).map(|held| parse_windows(held)).transpose()?.map_or(0, |held| held.len()); + Ok(format!("chat-purge requested: {} {}..{} request={:?} windows={windows}", purge.room_id, purge.from, purge.to, purge.request)) +} diff --git a/crates/huginn-daemon/src/sources/job_tests.rs b/crates/huginn-daemon/src/sources/job_tests.rs new file mode 100644 index 0000000..7433dfa --- /dev/null +++ b/crates/huginn-daemon/src/sources/job_tests.rs @@ -0,0 +1,287 @@ +//! One run of the job: forge refresh and chat refresh over a fake Synapse reached through the real client, +//! a fake vector store and the fake embedder; the exit status; the chat-off retirement; the purge request. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use anyhow::{Result, bail}; +use chrono::{DateTime, Utc}; +use serde_json::json; + +use super::chat::{CHAT_PURGE_PREFIX, RoomHistory}; +use super::job::{Chat, Purge, purge, run}; +use super::matrix::HttpMatrix; +use super::matrix_tests::{Room, Rooms, TOKEN, message, state, synapse}; +use super::room_scope::Estate; +use super::tests::{FakeForge, Remote}; +use super::test_http::forge_of; +use super::*; +use crate::index::fakes::{FakeEmbedder, FakeSources}; + +const DAY0: u64 = 1_794_268_800_000; // 2026-11-10T00:00:00Z + +fn now() -> DateTime { + DateTime::parse_from_rfc3339("2026-11-17T00:00:00Z").unwrap().with_timezone(&Utc) +} + +fn estate() -> Estate { + Estate::new(["!e1:s", "!e2:s", "!e3:s"].map(str::to_owned).into()).unwrap() +} + +struct Out { + said: Vec, + complained: Vec, + status: u8, +} + +struct Rig { + embedder: FakeEmbedder, + index: FakeSources, + forge: FakeForge, + _remote: Remote, + root: tempfile::TempDir, +} + +impl Rig { + fn new() -> Self { + let remote = Remote::new(); + remote.commit(&[("README.md", b"# Demo\n")], &[]); + Self { embedder: FakeEmbedder::new("d1"), index: FakeSources::default(), forge: FakeForge::of(&[("Demo", &remote)]), _remote: remote, root: tempfile::tempdir().unwrap() } + } + + fn go(&mut self, history: Option<&mut H>) -> Out { + let (mut said, mut complained) = (Vec::new(), Vec::new()); + let estate = estate(); + let chat = history.map(|history| Chat { history, estate: &estate }); + let status = run(&mut self.embedder, &mut self.index, &mut self.forge, self.root.path(), chat, &mut |line| said.push(line), &mut |line| complained.push(line)); + Out { said, complained, status } + } + + fn chat_bodies(&self) -> Vec { + let mut bodies: Vec = self.index.state.lock().unwrap().chat.values().flat_map(|point| point.payload.messages.iter().map(|m| m.body.clone())).collect(); + bodies.sort(); + bodies + } + + fn chat_rooms(&self) -> std::collections::BTreeSet { + self.index.state.lock().unwrap().chat.values().map(|point| point.payload.room_id.clone()).collect() + } +} + +fn rooms(list: Vec<(&str, Vec, Vec)>) -> Rooms { + Arc::new(Mutex::new(list.into_iter().map(|(id, state, timeline)| (id.to_owned(), Room { state, timeline })).collect::>())) +} + +fn platform() -> String { + ["disc", "ord"].concat() +} + +fn world() -> Rooms { + let relay = format!("@_{}_1:s", platform()); + rooms(vec![ + ("!swarm:s", vec![state("m.room.canonical_alias", json!({ "alias": "#swarm:gamecult.org" })), state("m.room.join_rules", json!({ "join_rule": "invite" }))], vec![ + message("$m1", "@a:s", DAY0 - 5000, "first words"), + message("$m2", "@b:s", DAY0 - 4000, "to be redacted"), + message("$m3", "@a:s", DAY0 + 1000, "inside the window"), + ]), + ("!bridged:s", vec![state("m.bridge", json!({}))], vec![message("$b1", "@a:s", DAY0, "bridged talk")]), + ("!relay:s", vec![], vec![message("$d1", &relay, DAY0, "relayed talk")]), + ("!aliased:s", vec![state("m.room.canonical_alias", json!({ "alias": "#Users:gamecult.org" }))], vec![message("$u1", "@a:s", DAY0, "estate talk")]), + ]) +} + +fn all_lines(out: &Out) -> String { + out.said.iter().chain(&out.complained).cloned().collect::>().join("\n") +} + +#[test] +fn a_full_run_over_a_fake_synapse() { + let world = world(); + let (base, asked) = synapse(&world); + let mut matrix = HttpMatrix::new(&base, TOKEN); + let mut rig = Rig::new(); + + let first = rig.go(Some(&mut matrix)); + assert_eq!(first.status, 1, "joined bridged, relay and estate rooms are refused: {:?}", first.complained); + assert_eq!(rig.chat_rooms(), ["!swarm:s".to_owned()].into(), "an invite-only room is ingested; nothing of the refused rooms is"); + assert_eq!(rig.chat_bodies(), ["first words", "inside the window", "to be redacted"]); + for refused in ["!bridged:s", "!relay:s", "!aliased:s"] { + assert!(first.complained.iter().any(|line| line.starts_with("refused:") && line.contains(refused)), "{refused}: {:?}", first.complained); + } + assert!(!rig.embedder.texts().iter().any(|text| text.contains("bridged talk") || text.contains("relayed talk") || text.contains("estate talk"))); + assert!(first.said.iter().any(|line| line.starts_with("Demo ")), "the forge's line is journaled too"); + assert!(first.said.iter().any(|line| line.starts_with("#swarm:gamecult.org (!swarm:s)") && line.contains("events=3")), "{:?}", first.said); + assert!(asked.seen().iter().all(|seen| seen.method == "GET")); + + // The steward asks for the window; then the room edits, redacts and leaks a key. + let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "follow_up:x", now()).unwrap(); + assert_eq!(purge(&mut rig.index, &request).unwrap(), "chat-purge requested: !swarm:s 2026-11-10T00:00:00Z..2026-11-11T00:00:00Z request=\"follow_up:x\" windows=1"); + let before = rig.index.cursors(); + assert_eq!(purge(&mut rig.index, &request).unwrap().ends_with("windows=1"), true); + assert_eq!(rig.index.cursors(), before, "a repeated request writes nothing"); + let secret = format!("key ghp_{}", "k2L9".repeat(9)); + { + let mut held = world.lock().unwrap(); + let room = held.get_mut("!swarm:s").unwrap(); + room.timeline.push(json!({ "type": "m.room.message", "event_id": "$edit", "sender": "@a:s", "origin_server_ts": DAY0 + 90_000_000, "content": { + "msgtype": "m.text", "body": "* first words, amended", "m.new_content": { "msgtype": "m.text", "body": "first words, amended" }, "m.relates_to": { "rel_type": "m.replace", "event_id": "$m1" } } })); + room.timeline.push(json!({ "type": "m.room.redaction", "event_id": "$red", "sender": "@b:s", "origin_server_ts": DAY0 + 91_000_000, "content": { "redacts": "$m2" } })); + room.timeline.push(message("$m4", "@a:s", DAY0 + 92_000_000, &secret)); + } + let second = rig.go(Some(&mut matrix)); + assert_eq!(rig.chat_bodies().len(), 2, "{:?}", rig.chat_bodies()); + let bodies = rig.chat_bodies().join("\n"); + assert!(bodies.contains("first words, amended") && !bodies.contains("to be redacted") && !bodies.contains("inside the window"), "{bodies}"); + assert!(!bodies.contains("ghp_k2L9") && !rig.embedder.texts().iter().any(|text| text.contains("ghp_k2L9")), "the key was scrubbed before it was embedded"); + let text = all_lines(&second); + assert!(second.said.iter().any(|line| line.contains("purged=")), "{text}"); + assert!(second.complained.iter().any(|line| line == "scrubbed: !swarm:s $m4 key-shape" || line.starts_with("scrubbed: !swarm:s $m4 ")), "{text}"); + for body in ["first words", "to be redacted", "inside the window", "relayed talk", "ghp_"] { + assert!(!text.contains(body), "the journal carries ids and counts, not bodies: {body}"); + } + let third = rig.go(Some(&mut matrix)); + assert!(!rig.chat_bodies().join("\n").contains("inside the window"), "a purged window does not come back"); + assert!(third.complained.iter().all(|line| line.starts_with("refused:")), "{:?}", third.complained); +} + +#[test] +fn no_chat_flags_purge_the_corpus_and_make_no_matrix_call() { + let world = world(); + let (base, _) = synapse(&world); + let mut matrix = HttpMatrix::new(&base, TOKEN); + let mut rig = Rig::new(); + rig.go(Some(&mut matrix)); + let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "r", now()).unwrap(); + purge(&mut rig.index, &request).unwrap(); + assert!(!rig.chat_bodies().is_empty()); + + let (hung, asked) = forge_of(|_| None::); + drop(hung); + let out = rig.go::(None); + assert_eq!(out.status, 0, "{:?}", out.complained); + assert!(rig.chat_bodies().is_empty() && rig.chat_rooms().is_empty()); + assert!(rig.index.cursors().keys().any(|key| key.starts_with(CHAT_PURGE_PREFIX)), "tombstones stay"); + assert!(!rig.index.cursors().keys().any(|key| key.starts_with("chat:")), "cursors go"); + assert!(asked.seen().is_empty()); +} + +struct BrokenForge; + +impl Forge for BrokenForge { + fn catalog(&mut self) -> Result { + bail!("the forge is down") + } + + fn clone_url(&self, _name: &str) -> String { + String::new() + } +} + +struct DeadSynapse; + +impl RoomHistory for DeadSynapse { + fn joined_rooms(&mut self) -> Result> { + bail!("synapse is down") + } + + fn facts(&mut self, _room_id: &str) -> Result { + bail!("synapse is down") + } + + fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result> { + bail!("synapse is down") + } +} + +#[test] +fn exit_status_is_clean_only_when_forge_and_chat_both_are() { + let clean = rooms(vec![("!swarm:s", vec![], vec![message("$m1", "@a:s", 1, "words")])]); + let (base, _) = synapse(&clean); + let mut rig = Rig::new(); + let out = rig.go(Some(&mut HttpMatrix::new(&base, TOKEN))); + assert_eq!((out.status, out.complained.len()), (0, 0)); + + // Scrub hits alone are not a failure; they are reported. + clean.lock().unwrap().get_mut("!swarm:s").unwrap().timeline.push(message("$m2", "@a:s", 2, &format!("ghp_{}", "k2L9".repeat(9)))); + let out = rig.go(Some(&mut HttpMatrix::new(&base, TOKEN))); + assert_eq!(out.status, 0, "{:?}", out.complained); + assert!(out.complained.iter().any(|line| line.starts_with("scrubbed: !swarm:s $m2"))); + + // A chat failure still reports the forge, and a forge failure still runs chat. + let out = rig.go(Some(&mut DeadSynapse)); + assert_eq!(out.status, 1); + assert!(out.complained.iter().any(|line| line.contains("synapse is down"))); + let (mut said, mut complained) = (Vec::new(), Vec::new()); + let (estate, mut ok) = (estate(), HttpMatrix::new(&base, TOKEN)); + let status = run(&mut rig.embedder, &mut rig.index, &mut BrokenForge, rig.root.path(), Some(Chat { history: &mut ok, estate: &estate }), &mut |l| said.push(l), &mut |l| complained.push(l)); + assert_eq!(status, 1); + assert!(complained.iter().any(|line| line.contains("the forge is down")), "{complained:?}"); + assert!(said.iter().any(|line| line.contains("(!swarm:s)")), "chat ran after the forge failed: {said:?}"); +} + +#[test] +fn a_request_cannot_forge_a_journal_line() { + let mut rig = Rig::new(); + rig.go::(None); + let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "x\nrefused: !forged:s", now()).unwrap(); + let line = purge(&mut rig.index, &request).unwrap(); + assert_eq!(line.lines().count(), 1, "{line:?}"); +} + +#[test] +fn a_failed_room_report_goes_to_the_complaint_stream_and_fails_the_run() { + let mut rig = Rig::new(); + struct FactsDown(Rooms); + impl RoomHistory for FactsDown { + fn joined_rooms(&mut self) -> Result> { + Ok(vec!["!swarm:s".to_owned()]) + } + fn facts(&mut self, _room_id: &str) -> Result { + bail!("facts are down") + } + fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result> { + bail!("unreachable") + } + } + let out = rig.go(Some(&mut FactsDown(rooms(vec![])))); + assert_eq!(out.status, 1); + assert!(out.complained.iter().any(|line| line.contains("!swarm:s") && line.contains("facts are down")), "{:?}", out.complained); + assert!(!out.said.iter().any(|line| line.contains("facts are down")), "{:?}", out.said); +} + +#[test] +fn chat_purge_makes_no_matrix_request() { + let world = world(); + let (base, asked) = synapse(&world); + let mut rig = Rig::new(); + rig.go(Some(&mut HttpMatrix::new(&base, TOKEN))); + let made = asked.seen().len(); + let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "r", now()).unwrap(); + purge(&mut rig.index, &request).unwrap(); + assert_eq!(asked.seen().len(), made); +} + +#[test] +fn a_retirement_failure_goes_to_the_complaint_stream() { + struct Left; + impl RoomHistory for Left { + fn joined_rooms(&mut self) -> Result> { + Ok(vec![]) + } + fn facts(&mut self, _room_id: &str) -> Result { + bail!("unreachable") + } + fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result> { + bail!("unreachable") + } + } + let world = rooms(vec![("!swarm:s", vec![], vec![message("$m1", "@a:s", 1, "words")])]); + let (base, _) = synapse(&world); + let mut rig = Rig::new(); + rig.go(Some(&mut HttpMatrix::new(&base, TOKEN))); + rig.index.state.lock().unwrap().fail_chat_deletes = true; + let out = rig.go(Some(&mut Left)); + assert_eq!(out.status, 1); + assert!(out.complained.iter().any(|line| line.starts_with("huginn-sources: retiring !swarm:s failed")), "{:?}", out.complained); + assert!(!out.said.iter().any(|line| line.contains("retiring")), "{:?}", out.said); +} diff --git a/crates/huginn-daemon/src/sources/matrix.rs b/crates/huginn-daemon/src/sources/matrix.rs new file mode 100644 index 0000000..a757ae4 --- /dev/null +++ b/crates/huginn-daemon/src/sources/matrix.rs @@ -0,0 +1,159 @@ +//! Synapse's client API as the reader user sees it: Matrix JSON in, `RoomEvent` +//! and `RoomFacts` out. JSON for Matrix is spoken here and nowhere else. This +//! client reads with GET only and holds no policy: it filters nothing, decides +//! no room's or sender's scope, and names no platform. What is in scope is +//! `room_scope`'s; the token is read at one place, the `Authorization` header. + +use std::time::Duration; + +use anyhow::{Context, Result, bail}; +use serde_json::Value; + +use super::chat::{EventKind, EventPage, RoomEvent, RoomHistory}; +use super::room_scope::RoomFacts; + +const CALL_TIMEOUT: Duration = Duration::from_secs(60); + +/// A bridge's state event types: either marks a room as relaying another platform. +const BRIDGE_TYPES: [&str; 2] = ["m.bridge", "uk.half-shot.bridge"]; + +pub struct HttpMatrix { + agent: ureq::Agent, + base: String, + token: String, +} + +impl std::fmt::Debug for HttpMatrix { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("HttpMatrix").field("base", &self.base).finish_non_exhaustive() + } +} + +/// A path segment with everything but unreserved characters escaped. +fn segment(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + for byte in raw.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + out.push(char::from(byte)); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +impl HttpMatrix { + pub fn new(base_url: &str, token: &str) -> Self { + let agent: ureq::Agent = + ureq::Agent::config_builder().timeout_global(Some(CALL_TIMEOUT)).http_status_as_error(false).build().into(); + Self { agent, base: base_url.trim_end_matches('/').to_owned(), token: token.to_owned() } + } + + /// One GET of `path` (already escaped): the status and the JSON body. No message made here carries the token. + fn get(&self, path: &str) -> Result<(u16, Value)> { + let url = format!("{}{path}", self.base); + let mut response = self + .agent + .get(&url) + .header("Authorization", format!("Bearer {}", self.token)) + .call() + .with_context(|| format!("no answer from {url}"))?; + let status = response.status().as_u16(); + let body = response.body_mut().read_to_string().with_context(|| format!("reading the answer from {url}"))?; + Ok((status, serde_json::from_str(&body).unwrap_or(Value::Null))) + } + + fn ok(&self, path: &str) -> Result { + let (status, body) = self.get(path)?; + if status != 200 { + bail!("synapse answered {status} for {path}"); + } + Ok(body) + } +} + +fn event_of(raw: &Value) -> Result { + let text = |key: &str| raw[key].as_str().map(str::to_owned).with_context(|| format!("an event has no {key}")); + let content = &raw["content"]; + let content_keys = content.as_object().map(|map| map.keys().cloned().collect()).unwrap_or_default(); + Ok(RoomEvent { event_id: text("event_id")?, sender: text("sender")?, ts_ms: raw["origin_server_ts"].as_u64().context("an event has no origin_server_ts")?, content_keys, kind: kind_of(raw) }) +} + +fn kind_of(raw: &Value) -> EventKind { + let content = &raw["content"]; + if !raw["unsigned"]["redacted_because"].is_null() { + return EventKind::Other; + } + match raw["type"].as_str() { + Some("m.room.redaction") => match content["redacts"].as_str().or_else(|| raw["redacts"].as_str()) { + Some(target) => EventKind::Redaction { target: target.to_owned() }, + None => EventKind::Other, + }, + Some("m.room.message") if matches!(content["msgtype"].as_str(), Some("m.text" | "m.notice" | "m.emote")) => { + let Some(body) = content["body"].as_str() else { return EventKind::Other }; + let relation = &content["m.relates_to"]; + if relation["rel_type"].as_str() == Some("m.replace") { + return match (relation["event_id"].as_str(), content["m.new_content"]["body"].as_str()) { + (Some(target), Some(new)) => EventKind::Edit { target: target.to_owned(), body: new.to_owned() }, + _ => EventKind::Other, + }; + } + EventKind::Message { body: body.to_owned() } + } + _ => EventKind::Other, + } +} + +impl RoomHistory for HttpMatrix { + fn joined_rooms(&mut self) -> Result> { + let body = self.ok("/_matrix/client/v3/joined_rooms")?; + let rooms = body["joined_rooms"].as_array().context("the joined rooms answer is not a list")?; + Ok(rooms.iter().filter_map(|room| room.as_str().map(str::to_owned)).collect()) + } + + fn facts(&mut self, room_id: &str) -> Result { + let body = self.ok(&format!("/_matrix/client/v3/rooms/{}/state", segment(room_id)))?; + let state = body.as_array().context("the room state is not a list")?; + let mut facts = RoomFacts { room_id: room_id.to_owned(), alias: None, alt_aliases: Vec::new(), name: None, encrypted: false, direct: false, bridge: None }; + for event in state { + let content = &event["content"]; + match event["type"].as_str() { + Some("m.room.canonical_alias") => { + facts.alias = content["alias"].as_str().map(str::to_owned); + facts.alt_aliases = content["alt_aliases"].as_array().map(|list| list.iter().filter_map(|alias| alias.as_str().map(str::to_owned)).collect()).unwrap_or_default(); + } + Some("m.room.name") => facts.name = content["name"].as_str().map(str::to_owned), + Some("m.room.encryption") => facts.encrypted = true, + Some("m.room.member") if content["is_direct"].as_bool() == Some(true) => facts.direct = true, + Some(kind) if BRIDGE_TYPES.contains(&kind) => { + facts.bridge.get_or_insert_with(|| kind.to_owned()); + } + _ => {} + } + } + Ok(facts) + } + + fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result> { + let room = segment(room_id); + let from = match after { + None => String::new(), + Some(event) => { + let (status, body) = self.get(&format!("/_matrix/client/v3/rooms/{room}/context/{}?limit=0", segment(event)))?; + match status { + 200 => match body["end"].as_str() { + Some(end) => format!("&from={}", segment(end)), + None => bail!("synapse gave no end token for the context of {event} in {room_id}"), + }, + 404 => return Ok(None), + other => bail!("synapse answered {other} for the context of {event} in {room_id}"), + } + } + }; + let body = self.ok(&format!("/_matrix/client/v3/rooms/{room}/messages?dir=f&limit={limit}{from}"))?; + let chunk = body["chunk"].as_array().context("the messages answer has no chunk")?; + let events = chunk.iter().map(event_of).collect::>>()?; + let more = body["end"].as_str().is_some() && !events.is_empty(); + Ok(Some(EventPage { events, more })) + } +} diff --git a/crates/huginn-daemon/src/sources/matrix_tests.rs b/crates/huginn-daemon/src/sources/matrix_tests.rs new file mode 100644 index 0000000..20ff3a9 --- /dev/null +++ b/crates/huginn-daemon/src/sources/matrix_tests.rs @@ -0,0 +1,207 @@ +//! The Matrix client against a fake Synapse that speaks the client API's JSON. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use serde_json::{Value, json}; + +use super::chat::{EventKind, RoomHistory}; +use super::matrix::HttpMatrix; +use super::test_http::{Asked, Reply, forge_of}; + +pub(super) const TOKEN: &str = "syt_reader_9f8e7d6c5b4a"; + +/// A room as the fake server holds it: its state events and its timeline. +pub(super) struct Room { + pub state: Vec, + pub timeline: Vec, +} + +pub(super) type Rooms = Arc>>; + +pub(super) fn decode(raw: &str) -> String { + let bytes = raw.as_bytes(); + let mut out = Vec::new(); + let mut at = 0; + while at < bytes.len() { + if bytes[at] == b'%' { + out.push(u8::from_str_radix(&raw[at + 1..at + 3], 16).unwrap()); + at += 3; + } else { + out.push(bytes[at]); + at += 1; + } + } + String::from_utf8(out).unwrap() +} + +pub(super) fn message(id: &str, sender: &str, ts: u64, body: &str) -> Value { + json!({ "type": "m.room.message", "event_id": id, "sender": sender, "origin_server_ts": ts, "content": { "msgtype": "m.text", "body": body } }) +} + +pub(super) fn state(kind: &str, content: Value) -> Value { + json!({ "type": kind, "state_key": "", "content": content }) +} + +/// A Synapse over `rooms`, joined to the rooms in the map: GET only, bearer required. +pub(super) fn synapse(rooms: &Rooms) -> (String, Asked) { + let rooms = Arc::clone(rooms); + forge_of(move |raw| { + let (path, query) = raw.split_once('?').unwrap_or((raw, "")); + let path = decode(path); + let rooms = rooms.lock().unwrap(); + let rest = path.strip_prefix("/_matrix/client/v3/")?; + if rest == "joined_rooms" { + return Some(Reply::from(json!({ "joined_rooms": rooms.keys().collect::>() }).to_string())); + } + let rest = rest.strip_prefix("rooms/")?; + let (id, rest) = rest.split_once('/')?; + let Some(room) = rooms.get(id) else { return Some(Reply::status(404, r#"{"errcode":"M_NOT_FOUND"}"#)) }; + let query: BTreeMap = query.split('&').filter_map(|pair| pair.split_once('=')).map(|(k, v)| (k.to_owned(), decode(v))).collect(); + if rest == "state" { + return Some(Reply::from(Value::Array(room.state.clone()).to_string())); + } + if let Some(event) = rest.strip_prefix("context/") { + return Some(match room.timeline.iter().any(|held| held["event_id"] == event) { + true => Reply::from(json!({ "end": format!("t_{event}"), "events_before": [], "events_after": [] }).to_string()), + false => Reply::status(404, r#"{"errcode":"M_NOT_FOUND"}"#), + }); + } + if rest == "messages" { + let start = match query.get("from") { + Some(from) => 1 + room.timeline.iter().position(|held| format!("t_{}", held["event_id"].as_str().unwrap()) == *from)?, + None => 0, + }; + let limit: usize = query["limit"].parse().unwrap(); + let chunk: Vec<&Value> = room.timeline.iter().skip(start).take(limit).collect(); + let end = chunk.last().map(|last| format!("t_{}", last["event_id"].as_str().unwrap())); + return Some(Reply::from(json!({ "chunk": chunk, "end": end }).to_string())); + } + None + }) +} + +fn one_room(state: Vec, timeline: Vec) -> Rooms { + Arc::new(Mutex::new(BTreeMap::from([("!r:s".to_owned(), Room { state, timeline })]))) +} + +#[test] +fn state_maps_to_facts_and_the_join_rule_is_not_read() { + let plain = one_room(vec![state("m.room.join_rules", json!({ "join_rule": "invite" }))], vec![]); + let (base, _) = synapse(&plain); + let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap(); + assert_eq!((facts.alias, facts.name, facts.encrypted, facts.direct, facts.bridge), (None, None, false, false, None)); + for rule in ["public", "knock", "invite"] { + let rooms = one_room(vec![state("m.room.join_rules", json!({ "join_rule": rule }))], vec![]); + let (base, _) = synapse(&rooms); + let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap(); + assert!(!facts.direct && !facts.encrypted && facts.bridge.is_none(), "{rule}"); + } + let full = one_room( + vec![ + state("m.room.canonical_alias", json!({ "alias": "#a:s", "alt_aliases": ["#b:s", "#c:s"] })), + state("m.room.name", json!({ "name": "Swarm" })), + state("m.room.encryption", json!({ "algorithm": "m.megolm.v1.aes-sha2" })), + json!({ "type": "m.room.member", "state_key": "@x:s", "content": { "membership": "join" } }), + json!({ "type": "m.room.member", "state_key": "@y:s", "content": { "membership": "join", "is_direct": true } }), + state("uk.half-shot.bridge", json!({})), + ], + vec![], + ); + let (base, _) = synapse(&full); + let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap(); + assert_eq!(facts.alias.as_deref(), Some("#a:s")); + assert_eq!(facts.alt_aliases, ["#b:s", "#c:s"]); + assert_eq!(facts.name.as_deref(), Some("Swarm")); + assert!(facts.encrypted && facts.direct); + assert_eq!(facts.bridge.as_deref(), Some("uk.half-shot.bridge")); + let relay = one_room(vec![state("m.bridge", json!({}))], vec![]); + let (base, _) = synapse(&relay); + assert_eq!(HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap().bridge.as_deref(), Some("m.bridge")); +} + +#[test] +fn events_map_to_kinds_and_content_keys_are_names_only() { + let mut other = message("$img", "@a:s", 3, "pic.png"); + other["content"]["msgtype"] = json!("m.image"); + let mut pruned = message("$pruned", "@a:s", 4, "was here"); + pruned["unsigned"] = json!({ "redacted_because": { "event_id": "$x" } }); + let mut notice = message("$n", "@a:s", 5, "n"); + notice["content"]["msgtype"] = json!("m.notice"); + let mut emote = message("$e", "@a:s", 6, "e"); + emote["content"]["msgtype"] = json!("m.emote"); + let edit = json!({ "type": "m.room.message", "event_id": "$edit", "sender": "@a:s", "origin_server_ts": 7, "content": { + "msgtype": "m.text", "body": "* new", "m.new_content": { "msgtype": "m.text", "body": "new" }, "m.relates_to": { "rel_type": "m.replace", "event_id": "$m1" } } }); + let redaction = json!({ "type": "m.room.redaction", "event_id": "$red", "sender": "@a:s", "origin_server_ts": 8, "content": { "redacts": "$m1" } }); + let old_redaction = json!({ "type": "m.room.redaction", "event_id": "$red2", "sender": "@a:s", "origin_server_ts": 9, "redacts": "$m2", "content": {} }); + let encrypted = json!({ "type": "m.room.encrypted", "event_id": "$enc", "sender": "@a:s", "origin_server_ts": 10, "content": { "ciphertext": "xx" } }); + let timeline = vec![message("$m1", "@a:s", 1, "hello"), other, pruned, notice, emote, edit, redaction, old_redaction, encrypted]; + let rooms = one_room(vec![], timeline); + let (base, _) = synapse(&rooms); + let page = HttpMatrix::new(&base, TOKEN).events_after("!r:s", None, 50).unwrap().unwrap(); + let kinds: Vec = page.events.iter().map(|event| event.kind.clone()).collect(); + assert_eq!( + kinds, + [ + EventKind::Message { body: "hello".into() }, + EventKind::Other, + EventKind::Other, + EventKind::Message { body: "n".into() }, + EventKind::Message { body: "e".into() }, + EventKind::Edit { target: "$m1".into(), body: "new".into() }, + EventKind::Redaction { target: "$m1".into() }, + EventKind::Redaction { target: "$m2".into() }, + EventKind::Other, + ] + ); + assert_eq!(page.events[0].content_keys, ["body", "msgtype"]); + assert_eq!(page.events[5].content_keys, ["body", "m.new_content", "m.relates_to", "msgtype"], "names, never values"); + assert_eq!((page.events[0].event_id.as_str(), page.events[0].sender.as_str(), page.events[0].ts_ms), ("$m1", "@a:s", 1)); +} + +#[test] +fn resume_uses_the_cursor_events_end_token_and_a_missing_event_is_none() { + let rooms = one_room(vec![], vec![message("$1", "@a:s", 1, "a"), message("$2", "@a:s", 2, "b"), message("$3", "@a:s", 3, "c")]); + let (base, asked) = synapse(&rooms); + let mut matrix = HttpMatrix::new(&base, TOKEN); + let page = matrix.events_after("!r:s", Some("$1"), 1).unwrap().unwrap(); + assert_eq!(page.events.iter().map(|event| event.event_id.as_str()).collect::>(), ["$2"]); + assert!(page.more); + assert_eq!(asked.paths(), ["/_matrix/client/v3/rooms/%21r%3As/context/%241?limit=0", "/_matrix/client/v3/rooms/%21r%3As/messages?dir=f&limit=1&from=t_%241"]); + assert!(matrix.events_after("!r:s", Some("$unknown"), 5).unwrap().is_none()); + let tail = matrix.events_after("!r:s", Some("$3"), 5).unwrap().unwrap(); + assert!(tail.events.is_empty() && !tail.more, "an empty chunk ends the run"); + let first = matrix.events_after("!r:s", None, 2).unwrap().unwrap(); + assert_eq!(first.events.len(), 2); + assert_eq!(asked.paths().last().unwrap(), "/_matrix/client/v3/rooms/%21r%3As/messages?dir=f&limit=2", "no from, no context call"); +} + +#[test] +fn only_get_always_the_bearer_and_the_token_is_never_printed() { + let rooms = one_room(vec![], vec![message("$1", "@a:s", 1, "a")]); + let (base, asked) = synapse(&rooms); + let mut matrix = HttpMatrix::new(&base, TOKEN); + matrix.joined_rooms().unwrap(); + matrix.facts("!r:s").unwrap(); + matrix.events_after("!r:s", Some("$1"), 3).unwrap(); + let seen = asked.seen(); + assert!(seen.len() >= 4 && seen.iter().all(|seen| seen.method == "GET" && seen.authorization.as_deref() == Some(&format!("Bearer {TOKEN}"))), "{seen:?}"); + let (base, _) = forge_of(|_| Some(Reply::status(401, &format!(r#"{{"errcode":"M_UNKNOWN_TOKEN","error":"bad {TOKEN}"}}"#)))); + let mut matrix = HttpMatrix::new(&base, TOKEN); + let error = matrix.joined_rooms().unwrap_err(); + assert!(format!("{error:#}").contains("401")); + assert!(!format!("{error:#}{error:?}").contains(TOKEN)); + assert!(!format!("{matrix:?}").contains(TOKEN)); +} + +#[test] +fn ids_are_percent_encoded_in_paths() { + let weird = "$a+b/c=d:s"; + let rooms: Rooms = Arc::new(Mutex::new(BTreeMap::from([("!ro om:s".to_owned(), Room { state: vec![], timeline: vec![message(weird, "@a:s", 1, "x"), message("$2", "@a:s", 2, "y")] })]))); + let (base, asked) = synapse(&rooms); + let mut matrix = HttpMatrix::new(&base, TOKEN); + let page = matrix.events_after("!ro om:s", Some(weird), 5).unwrap().unwrap(); + assert_eq!(page.events.len(), 1, "the server decoded the room and event ids to the originals"); + let raw = asked.paths(); + assert!(raw.iter().all(|path| !path.contains('+') && !path.contains(' ') && !path.contains("!ro")), "{raw:?}"); +} diff --git a/crates/huginn-daemon/src/sources/mod.rs b/crates/huginn-daemon/src/sources/mod.rs index 5e4133d..d698bb2 100644 --- a/crates/huginn-daemon/src/sources/mod.rs +++ b/crates/huginn-daemon/src/sources/mod.rs @@ -30,11 +30,20 @@ pub mod chat_purge; pub mod chunk; pub mod forge; pub mod git; +pub mod job; +pub mod matrix; pub mod room_scope; +#[cfg(test)] +mod test_http; + #[cfg(test)] mod chat_tests; #[cfg(test)] +mod job_tests; +#[cfg(test)] +mod matrix_tests; +#[cfg(test)] mod tests; use std::collections::hash_map::Entry; diff --git a/crates/huginn-daemon/src/sources/test_http.rs b/crates/huginn-daemon/src/sources/test_http.rs new file mode 100644 index 0000000..266fed7 --- /dev/null +++ b/crates/huginn-daemon/src/sources/test_http.rs @@ -0,0 +1,92 @@ +//! The one fake HTTP server the source tests talk to (the forge's and Matrix's): +//! it answers by path, holds open a path it has no answer for as a hung server +//! does, and records each request's method, path and `Authorization` header. + +use std::io::{Read, Write}; +use std::sync::{Arc, Mutex}; + +use serde_json::json; + +/// What a fake server answers: a status, the `Location` header when there is one, and a body. +pub struct Reply { + pub status: u16, + pub location: Option, + pub body: String, +} + +impl From for Reply { + fn from(body: String) -> Self { + Self { status: 200, location: None, body } + } +} + +impl Reply { + /// What the live forge answers an anonymous reader for a private repo. + pub fn not_found() -> Self { + Self { status: 404, location: None, body: json!({ "message": "The target couldn't be found." }).to_string() } + } + + /// What the live forge answers for a renamed repo: a 301 to its public address, `/forge` prefix and all. + pub fn moved_to(location: &str) -> Self { + Self { status: 301, location: Some(location.to_owned()), body: String::new() } + } + + pub fn status(status: u16, body: &str) -> Self { + Self { status, location: None, body: body.to_owned() } + } +} + +/// One request as the server saw it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Seen { + pub method: String, + pub path: String, + pub authorization: Option, +} + +/// Every request a fake server has seen, in order. +#[derive(Clone, Default)] +pub struct Asked(Arc>>); + +impl Asked { + pub fn seen(&self) -> Vec { + self.0.lock().unwrap().clone() + } + + pub fn paths(&self) -> Vec { + self.0.lock().unwrap().iter().map(|seen| seen.path.clone()).collect() + } +} + +/// Answers each request by its path from `routes`; a path it has no answer for is held open and never answered. +pub fn forge_of>(routes: impl Fn(&str) -> Option + Send + 'static) -> (String, Asked) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let asked = Asked::default(); + let record = asked.clone(); + std::thread::spawn(move || { + let mut held = Vec::new(); + for stream in listener.incoming() { + let mut stream = stream.unwrap(); + let mut buffer = [0_u8; 4096]; + let read = stream.read(&mut buffer).unwrap(); + let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); + let mut words = request.split_whitespace(); + let method = words.next().unwrap_or_default().to_owned(); + let path = words.next().unwrap_or_default().to_owned(); + let authorization = request + .lines() + .find_map(|line| line.split_once(':').filter(|(name, _)| name.eq_ignore_ascii_case("authorization"))) + .map(|(_, value)| value.trim().to_owned()); + record.0.lock().unwrap().push(Seen { method, path: path.clone(), authorization }); + match routes(&path).map(Into::into) { + Some(Reply { status, location, body }) => { + let location = location.map(|location| format!("location: {location}\r\n")).unwrap_or_default(); + write!(stream, "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\n{location}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap(); + } + None => held.push(stream), + } + } + }); + (base, asked) +} diff --git a/crates/huginn-daemon/src/sources/tests.rs b/crates/huginn-daemon/src/sources/tests.rs index a7203a4..017fdbf 100644 --- a/crates/huginn-daemon/src/sources/tests.rs +++ b/crates/huginn-daemon/src/sources/tests.rs @@ -25,23 +25,23 @@ fn git_in(dir: &Path, args: &[&str]) -> String { } /// A working repo on disk, committed to by the test. -struct Remote { +pub(super) struct Remote { dir: tempfile::TempDir, } impl Remote { - fn new() -> Self { + pub(super) fn new() -> Self { let dir = tempfile::tempdir().unwrap(); git_in(dir.path(), &["init", "--quiet", "-b", "main"]); Self { dir } } - fn path(&self) -> PathBuf { + pub(super) fn path(&self) -> PathBuf { self.dir.path().to_path_buf() } /// Writes files, removes paths, commits everything and returns the commit. - fn commit(&self, write: &[(&str, &[u8])], remove: &[&str]) -> String { + pub(super) fn commit(&self, write: &[(&str, &[u8])], remove: &[&str]) -> String { for (path, bytes) in write { let full = self.dir.path().join(path); std::fs::create_dir_all(full.parent().unwrap()).unwrap(); @@ -56,13 +56,13 @@ impl Remote { } } -struct FakeForge { +pub(super) struct FakeForge { repos: Vec<(SourceRepo, PathBuf)>, shortfall: Option, } impl FakeForge { - fn of(repos: &[(&str, &Remote)]) -> Self { + pub(super) fn of(repos: &[(&str, &Remote)]) -> Self { Self { repos: repos .iter() diff --git a/docs/chat-retrieval-map.md b/docs/chat-retrieval-map.md index bee6079..02627a2 100644 --- a/docs/chat-retrieval-map.md +++ b/docs/chat-retrieval-map.md @@ -120,10 +120,18 @@ k=60 are calibration, not design; RRF over a handful of lists is the established baseline and I did not survey learned fusion (it needs training data we lack). -## Status (head-maintainer pass, 2026-11-05, Huginn `25199c9`) +## Status (2026-11-17) -Checked against main on merge. Nothing in this map is built: no `texts` on -`SourceQuery`, no chat code under `sources/`. The anchors above were corrected. +Built: cut 1 (`texts` on `SourceQuery`), chat search and window, the chat ingest +core (`sources/chat.rs`, `room_scope.rs`, `chat_purge.rs`) and the ingest job. +The job is `huginn-sources`: with `--matrix-url`, `--matrix-token-file`, three +`--chat-exclude-room` ids (the estate rooms) and `--lock-file` it runs the forge +refresh and then `refresh_chat` through `sources/matrix.rs` (GET only, bearer +from a systemd credential file); without them it retires every chat point. The +rooms indexed are the reader's memberships; no flag lists rooms. `huginn-sources +chat-purge` writes a steward's purge tombstone under the same lock. What the +operator still has to do (reader account, credential, roster, unit lines) is the +handoff list of cut `gamecult-chat:cut_spec:cut-chat-ingest-job`. What changed since the map was written: - Questions 1 and 2 are answered with the recommendations