Chat ingest job: huginn-sources chat flags, Matrix reader, lock, chat-purge (#67)

This commit is contained in:
Eureka 2026-11-17 01:52:38 +00:00 • committed by Eureka
commit 07942c1a83
13 changed files with 1267 additions and 143 deletions

View file

@ -18,7 +18,7 @@ name = "huginn-daemon"
path = "src/main.rs"
# The source index's one writer: a oneshot job that brings `huginn_sources` to
# the forge's current state and exits. No daemon calls any of its writes.
# the forge's and the chat reader's current state and exits. No daemon calls any of its writes.
[[bin]]
name = "huginn-sources"
path = "src/bin/huginn-sources.rs"
@ -34,7 +34,8 @@ rmp-serde = "1"
rustix = { version = "1", default-features = false, features = ["std", "net"] }
serde = { version = "1", features = ["derive"] }
# The vector store's and the embedder's HTTP APIs are the xenos boundary, and
# JSON exists in `index/ollama.rs` and `index/qdrant.rs` only. `ureq` 3 with
# JSON exists in `index/ollama.rs`, `index/qdrant.rs`, `sources/forge.rs` and
# `sources/matrix.rs` only. `ureq` 3 with
# default features off is plain HTTP with no TLS (`rustls` and `gzip` are its
# only defaults), which is all the private mesh needs. It stays below 3.4,
# whose `base64` 0.23 would put a second `base64` in a tree that has one.

View file

@ -1,21 +1,38 @@
//! The source index's refresh job: one run brings the `huginn_sources`
//! collection to the forge's current state, journals one line per repo, and
//! exits nonzero if any repo failed or the collection could not be used.
//! The source index's refresh job: one run brings the forge corpus and, when
//! configured, the rooms the chat reader has joined (minus the estate and DM
//! exclusion) to current state, journals one line per repo and room, and exits
//! nonzero if either failed or the collection could not be used. The
//! `chat-purge` subcommand writes a steward's purge tombstone and nothing else.
//!
//! Exit status: 0 clean, 1 a run failed, 2 bad arguments, 3 another run holds
//! the lock. The Matrix token is read from a file (a systemd credential),
//! never from the environment or argv.
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::collections::{BTreeMap, BTreeSet};
use std::fs::{File, OpenOptions, TryLockError};
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use anyhow::{Context, Result, ensure};
use chrono::{DateTime, Utc};
use huginn_daemon::index::ollama::pool;
use huginn_daemon::index::qdrant::QdrantIndex;
use huginn_daemon::sources::forge::HttpForge;
use huginn_daemon::sources::refresh;
use huginn_daemon::sources::job::{self, Chat, Purge};
use huginn_daemon::sources::matrix::HttpMatrix;
use huginn_daemon::sources::room_scope::{ESTATE_ALIASES, Estate};
/// Where the forge answers when the flag is absent: the loopback address of the
/// forge on the host that runs this job.
const DEFAULT_FORGE_URL: &str = "http://127.0.0.1:3310";
/// The chat flags, which come all together or not at all.
struct ChatOptions {
matrix_url: String,
token: String,
estate: Estate,
}
struct Options {
state_root: PathBuf,
qdrant_url: String,
@ -25,75 +42,310 @@ struct Options {
reserve_urls: Vec<String>,
embedding_model: String,
forge_url: String,
chat: Option<ChatOptions>,
lock_file: Option<PathBuf>,
}
fn parse(args: impl Iterator<Item = String>) -> Result<Options> {
enum Command {
Run(Options),
Purge { qdrant_url: String, lock_file: PathBuf, purge: Purge },
}
/// How a run ends other than by finishing.
enum Stop {
Usage(anyhow::Error),
Locked,
}
/// The options named, each once, except those in `repeated`, which keep every value.
fn collect(args: impl Iterator<Item = String>, once: &[&str], repeated: &[&str]) -> Result<(BTreeMap<String, String>, BTreeMap<String, Vec<String>>)> {
let mut args = args;
let mut values: BTreeMap<String, String> = BTreeMap::new();
let mut ollama_urls = Vec::new();
let mut reserve_urls = Vec::new();
let mut singles = BTreeMap::new();
let mut many: BTreeMap<String, Vec<String>> = BTreeMap::new();
while let Some(name) = args.next() {
let name = name.strip_prefix("--").with_context(|| format!("expected --option, got {name:?}"))?;
ensure!(
matches!(name, "state-root" | "qdrant-url" | "ollama-url" | "reserve-ollama-url" | "embedding-model" | "forge-url"),
"unsupported huginn-sources option --{name}"
);
ensure!(once.contains(&name) || repeated.contains(&name), "unsupported huginn-sources option --{name}");
let value = args.next().with_context(|| format!("missing value for --{name}"))?;
if name == "ollama-url" || name == "reserve-ollama-url" {
ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}");
if name == "ollama-url" { ollama_urls.push(value) } else { reserve_urls.push(value) }
if repeated.contains(&name) {
many.entry(name.to_owned()).or_default().push(value);
} else {
ensure!(values.insert(name.to_owned(), value).is_none(), "duplicate option --{name}");
ensure!(singles.insert(name.to_owned(), value).is_none(), "duplicate option --{name}");
}
}
let take = |name: &str| values.get(name).cloned().with_context(|| format!("--{name} is required"));
let url = |name: &str, value: String| {
ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}");
Ok(value)
};
let state_root = PathBuf::from(take("state-root")?);
ensure!(state_root.is_absolute(), "--state-root must be an absolute path");
let embedding_model = take("embedding-model")?;
ensure!(!embedding_model.is_empty(), "--embedding-model must name a model");
Ok(Options {
state_root,
qdrant_url: url("qdrant-url", take("qdrant-url")?)?,
ollama_urls: {
ensure!(!ollama_urls.is_empty(), "at least one --ollama-url is required; repeat it for each GPU host");
ollama_urls
Ok((singles, many))
}
fn http(name: &str, value: String) -> Result<String> {
ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}");
Ok(value)
}
/// The Matrix reader's bearer travels in clear, and this build speaks no TLS: only a loopback host may be named.
fn loopback_http(name: &str, value: String) -> Result<String> {
let value = http(name, value)?;
let authority = value["http://".len()..].split(['/', '?', '#']).next().unwrap_or("");
let host = match authority.strip_prefix('[') {
// The bracket must end the authority or be followed by the port, so `[::1].evil.example` names no loopback.
Some(inner) => match inner.split_once(']') {
Some((host, "")) => host,
Some((host, port)) if port.starts_with(':') => host,
_ => "",
},
None => authority.split(':').next().unwrap_or(""),
};
let loopback = !authority.contains('@') && (host.eq_ignore_ascii_case("localhost") || host.parse::<std::net::IpAddr>().is_ok_and(|ip| ip.is_loopback()));
ensure!(loopback, "--{name} must name a loopback host (localhost, 127.0.0.0/8 or [::1]): the bearer token travels in clear and this build has no TLS, got {value:?}");
Ok(value)
}
fn absolute(name: &str, value: &str) -> Result<PathBuf> {
let path = PathBuf::from(value);
ensure!(path.is_absolute(), "--{name} must be an absolute path");
Ok(path)
}
fn chat_options(singles: &BTreeMap<String, String>, excluded: Vec<String>) -> Result<Option<ChatOptions>> {
let given = [singles.contains_key("matrix-url"), singles.contains_key("matrix-token-file"), !excluded.is_empty()];
if !given.contains(&true) {
return Ok(None);
}
for (present, name) in given.iter().zip(["matrix-url", "matrix-token-file", "chat-exclude-room"]) {
ensure!(*present, "--{name} is missing: the chat flags (--matrix-url, --matrix-token-file, --chat-exclude-room) come all together or not at all");
}
let ids: BTreeSet<String> = excluded.iter().cloned().collect();
ensure!(ids.len() == excluded.len(), "--chat-exclude-room must not repeat a room id");
ensure!(ids.len() == ESTATE_ALIASES.len(), "--chat-exclude-room must be given exactly {} times (the ids of {})", ESTATE_ALIASES.len(), ESTATE_ALIASES.join(", "));
ensure!(ids.iter().all(|id| id.starts_with('!')), "--chat-exclude-room takes room ids beginning with !, not aliases");
let path = absolute("matrix-token-file", &singles["matrix-token-file"])?;
let token = std::fs::read_to_string(&path).with_context(|| format!("reading --matrix-token-file {}", path.display()))?.trim().to_owned();
ensure!(!token.is_empty(), "--matrix-token-file {} is empty", path.display());
Ok(Some(ChatOptions { matrix_url: loopback_http("matrix-url", singles["matrix-url"].clone())?, token, estate: Estate::new(ids)? }))
}
fn parse(mut args: impl Iterator<Item = String>, now: DateTime<Utc>) -> Result<Command> {
let mut rest: Vec<String> = Vec::new();
let first = args.next();
if first.as_deref() == Some("chat-purge") {
let (singles, _) = collect(args, &["qdrant-url", "room", "from", "to", "request", "lock-file"], &[])?;
let take = |name: &str| singles.get(name).cloned().with_context(|| format!("--{name} is required"));
return Ok(Command::Purge {
qdrant_url: http("qdrant-url", take("qdrant-url")?)?,
lock_file: absolute("lock-file", &take("lock-file")?)?,
purge: Purge::check(&take("room")?, &take("from")?, &take("to")?, &take("request")?, now)?,
});
}
rest.extend(first);
let (mut singles, mut many) = collect(
rest.into_iter().chain(args),
&["state-root", "qdrant-url", "embedding-model", "forge-url", "matrix-url", "matrix-token-file", "lock-file"],
&["ollama-url", "reserve-ollama-url", "chat-exclude-room"],
)?;
let mut hosts = |name: &str| -> Result<Vec<String>> { many.remove(name).unwrap_or_default().into_iter().map(|url| http(name, url)).collect() };
let ollama_urls = hosts("ollama-url")?;
let reserve_urls = hosts("reserve-ollama-url")?;
let excluded = many.remove("chat-exclude-room").unwrap_or_default();
let take = |singles: &mut BTreeMap<String, String>, name: &str| singles.remove(name).with_context(|| format!("--{name} is required"));
let state_root = absolute("state-root", &take(&mut singles, "state-root")?)?;
let embedding_model = take(&mut singles, "embedding-model")?;
ensure!(!embedding_model.is_empty(), "--embedding-model must name a model");
ensure!(!ollama_urls.is_empty(), "at least one --ollama-url is required; repeat it for each GPU host");
let chat = chat_options(&singles, excluded)?;
let lock_file = singles.get("lock-file").map(|path| absolute("lock-file", path)).transpose()?;
ensure!(chat.is_none() || lock_file.is_some(), "--lock-file is required whenever the chat flags are given");
Ok(Command::Run(Options {
state_root,
qdrant_url: http("qdrant-url", take(&mut singles, "qdrant-url")?)?,
ollama_urls,
reserve_urls,
embedding_model,
forge_url: url("forge-url", values.get("forge-url").cloned().unwrap_or_else(|| DEFAULT_FORGE_URL.to_owned()))?,
})
forge_url: http("forge-url", singles.remove("forge-url").unwrap_or_else(|| DEFAULT_FORGE_URL.to_owned()))?,
chat,
lock_file,
}))
}
/// Takes the job's lock for the life of the returned file. A held lock means a run (or a
/// purge request) is in progress and nothing may change: the cursors object is replaced whole.
fn take_lock(path: &Path) -> Result<File, Stop> {
let file = OpenOptions::new().create(true).truncate(false).write(true).open(path).with_context(|| format!("opening --lock-file {}", path.display())).map_err(Stop::Usage)?;
match file.try_lock() {
Ok(()) => Ok(file),
Err(TryLockError::WouldBlock) => Err(Stop::Locked),
Err(TryLockError::Error(error)) => Err(Stop::Usage(anyhow::Error::new(error).context(format!("locking --lock-file {}", path.display())))),
}
}
fn lock_if(path: &Option<PathBuf>) -> Result<Option<File>, Stop> {
path.as_deref().map(take_lock).transpose()
}
fn execute(command: Command) -> Result<u8, Stop> {
match command {
Command::Purge { qdrant_url, lock_file, purge } => {
let _lock = take_lock(&lock_file)?;
let mut index = QdrantIndex::new(&qdrant_url);
Ok(match job::purge(&mut index, &purge) {
Ok(line) => {
println!("{line}");
0
}
Err(error) => {
eprintln!("huginn-sources: {error:#}");
1
}
})
}
Command::Run(options) => {
let _lock = lock_if(&options.lock_file)?;
let mut embedder = pool(&options.ollama_urls, &options.reserve_urls, &options.embedding_model);
let mut index = QdrantIndex::new(&options.qdrant_url);
let mut forge = HttpForge::new(&options.forge_url);
let mut matrix = options.chat.as_ref().map(|chat| HttpMatrix::new(&chat.matrix_url, &chat.token));
let chat = matrix.as_mut().zip(options.chat.as_ref()).map(|(history, chat)| Chat { history, estate: &chat.estate });
Ok(job::run(&mut embedder, &mut index, &mut forge, &options.state_root, chat, &mut |line| println!("{line}"), &mut |line| eprintln!("{line}")))
}
}
}
fn main() -> ExitCode {
let options = match parse(std::env::args().skip(1)) {
Ok(options) => options,
Err(error) => {
match parse(std::env::args().skip(1), Utc::now()).map_err(Stop::Usage).and_then(execute) {
Ok(status) => ExitCode::from(status),
Err(Stop::Usage(error)) => {
eprintln!("huginn-sources: {error:#}");
return ExitCode::from(2);
ExitCode::from(2)
}
};
let mut embedder = pool(&options.ollama_urls, &options.reserve_urls, &options.embedding_model);
let mut index = QdrantIndex::new(&options.qdrant_url);
let mut forge = HttpForge::new(&options.forge_url);
let mut journal = |report: &huginn_daemon::sources::Report| println!("{report}");
match refresh(&mut embedder, &mut index, &mut forge, &options.state_root, &mut journal) {
Ok(refreshed) if refreshed.is_clean() => ExitCode::SUCCESS,
Ok(refreshed) => {
if !refreshed.failed.is_empty() {
eprintln!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", "));
}
if let Some(why) = &refreshed.retirement_withheld {
eprintln!("huginn-sources: no repo was retired: {why}");
}
ExitCode::from(1)
}
Err(error) => {
eprintln!("huginn-sources: {error:#}");
ExitCode::from(1)
Err(Stop::Locked) => {
eprintln!("huginn-sources: huginn-sources is running");
ExitCode::from(3)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
const NOW: &str = "2026-11-17T00:00:00Z";
fn args(text: &str) -> impl Iterator<Item = String> {
text.split_whitespace().map(str::to_owned).collect::<Vec<_>>().into_iter()
}
fn now() -> DateTime<Utc> {
DateTime::parse_from_rfc3339(NOW).unwrap().with_timezone(&Utc)
}
fn error(text: &str) -> String {
match parse(args(text), now()) {
Ok(_) => panic!("parsed: {text}"),
Err(error) => format!("{error:#}"),
}
}
const BASE: &str = "--state-root /s --qdrant-url http://q --ollama-url http://o --embedding-model m";
const IDS: &str = "--chat-exclude-room !a:s --chat-exclude-room !b:s --chat-exclude-room !c:s";
fn with_token(secret: &str) -> (tempfile::TempDir, String) {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("token");
std::fs::write(&path, secret).unwrap();
let path = path.display().to_string();
(dir, path)
}
#[test]
fn chat_flags_come_all_or_none() {
let (_dir, token) = with_token(" sekret-token\n");
let lock = "--lock-file /l";
let full = format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS} {lock}");
let Ok(Command::Run(options)) = parse(args(&full), now()) else { panic!("full chat flags must parse") };
assert_eq!(options.chat.as_ref().unwrap().token, "sekret-token", "trimmed");
assert!(matches!(parse(args(BASE), now()), Ok(Command::Run(Options { chat: None, lock_file: None, .. }))));
assert!(error(&format!("{BASE} --matrix-url http://localhost {lock}")).contains("--matrix-token-file is missing"));
assert!(error(&format!("{BASE} --matrix-token-file {token} {IDS} {lock}")).contains("--matrix-url is missing"));
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {lock}")).contains("--chat-exclude-room is missing"));
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS}")).contains("--lock-file is required"));
}
#[test]
fn chat_flag_values_are_checked_and_never_print_the_token() {
let (_dir, token) = with_token("sekret-token");
let on = |extra: &str| format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} --lock-file /l {extra}");
assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !b:s")).contains("exactly 3"));
assert!(error(&on(&format!("{IDS} --chat-exclude-room !d:s"))).contains("exactly 3"));
assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !a:s --chat-exclude-room !b:s")).contains("repeat"));
assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !b:s --chat-exclude-room #c:s")).contains("beginning with !"));
assert!(error(&format!("{BASE} --matrix-url https://m --matrix-token-file {token} --lock-file /l {IDS}")).contains("plain http://"));
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file rel --lock-file /l {IDS}")).contains("absolute"));
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} --lock-file rel {IDS}")).contains("absolute"));
let (_empty, empty) = with_token(" \n");
let message = error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {empty} --lock-file /l {IDS}"));
assert!(message.contains(&empty) && message.contains("empty"));
let missing = error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file /nowhere/t --lock-file /l {IDS}"));
assert!(missing.contains("/nowhere/t"));
for text in [on(IDS), on("--chat-exclude-room !a:s")] {
assert!(!error(&format!("{text} --nope x")).contains("sekret-token"));
}
let dead = ["--chat", "room"].join("-");
assert!(error(&format!("{BASE} {dead} !x:s")).contains(&format!("unsupported huginn-sources option {dead}")));
}
fn purge(extra: &str) -> String {
error(&format!("chat-purge --qdrant-url http://q --lock-file /l {extra}"))
}
#[test]
fn chat_purge_arguments() {
let good = "--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z --request ref-1";
let Ok(Command::Purge { purge: asked, .. }) = parse(args(&format!("chat-purge --qdrant-url http://q --lock-file /l {good}")), now()) else { panic!() };
assert_eq!((asked.window.from_ms, asked.window.to_ms), (1_794_268_800_000, 1_794_355_200_000));
let bad = |from: &str, to: &str, room: &str, request: &str| purge(&format!("--room {room} --from {from} --to {to} --request {request}"));
let (f, t) = ("2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z");
assert!(bad(f, t, "#a:s", "r").contains("--room"));
assert!(bad("2026-11-10T00:00:00+00:00", t, "!r:s", "r").contains("--from"));
assert!(bad(f, "2026-11-11T00:00:00", "!r:s", "r").contains("--to"));
assert!(bad(t, f, "!r:s", "r").contains("before"));
assert!(bad(f, "2026-11-18T00:00:00Z", "!r:s", "r").contains("after now"));
assert!(bad("2025-11-09T00:00:00Z", t, "!r:s", "r").contains("366"));
assert!(purge(&format!("--room !r:s --from {f} --to {t} --request {}", "x".repeat(121))).contains("--request"));
assert!(purge("--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z").contains("--request is required"));
assert!(parse(args(&format!("chat-purge --qdrant-url http://q --lock-file /l {good} --matrix-url http://localhost")), now()).is_err(), "no Matrix flags");
}
#[test]
fn matrix_url_must_be_loopback_http() {
let (_dir, token) = with_token("sekret-token");
let with = |url: &str| format!("{BASE} --matrix-url {url} --matrix-token-file {token} --lock-file /l {IDS}");
for good in ["http://localhost", "http://127.0.0.1:8008", "http://127.9.9.9/", "http://[::1]:8008", "http://LOCALHOST:1/x"] {
assert!(parse(args(&with(good)), now()).is_ok(), "{good}");
}
for bad in ["http://matrix.example", "http://10.0.0.5:8008", "http://localhost@evil.example", "http://127.0.0.1.evil.example", "http://localhost.evil.example", "http://[::2]", "https://localhost", "http://127.0.0.1:pw@evil.example", "http://[::1].evil.example", "http://[::1]x"] {
assert!(error(&with(bad)).contains("--matrix-url"), "{bad}");
}
}
#[test]
fn a_held_lock_exits_3_through_execute_for_both_modes() {
let dir = tempfile::tempdir().unwrap();
let lock = dir.path().join("lock");
let held = take_lock(&lock).ok().unwrap();
let lock = lock.display().to_string();
let good = "--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z --request r";
let purge = parse(args(&format!("chat-purge --qdrant-url http://127.0.0.1:1 --lock-file {lock} {good}")), now()).unwrap();
assert!(matches!(execute(purge), Err(Stop::Locked)));
let (_t, token) = with_token("sekret-token");
let run = parse(args(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS} --lock-file {lock}")), now()).unwrap();
assert!(matches!(execute(run), Err(Stop::Locked)));
drop(held);
}
#[test]
fn a_held_lock_stops_the_run_untouched() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("lock");
let held = take_lock(&path).ok().unwrap();
assert!(matches!(take_lock(&path), Err(Stop::Locked)));
drop(held);
assert!(take_lock(&path).is_ok());
}
}

View file

@ -134,6 +134,39 @@ impl std::fmt::Display for ChatReport {
}
}
/// One line of a chat run's journal, typed so the job routes by what happened and not by how the line reads.
#[derive(Clone, Debug, PartialEq)]
pub enum Entry {
/// A room's report; a failed one is an alarm.
Report(ChatReport),
Refused(RefusedRoom),
Scrubbed(ScrubHit),
/// A failure outside any one room's report: retiring rooms.
Trouble(String),
}
impl Entry {
/// The operator must see this on stderr: a failure, a refusal, a scrub hit.
pub fn is_alarm(&self) -> bool {
match self {
Self::Report(report) => report.failure.is_some(),
Self::Refused(_) | Self::Scrubbed(_) | Self::Trouble(_) => true,
}
}
}
impl std::fmt::Display for Entry {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Report(report) => report.fmt(formatter),
Self::Refused(RefusedRoom { room_id, reason, event_id: None }) => write!(formatter, "refused: {room_id} {reason}"),
Self::Refused(RefusedRoom { room_id, reason, event_id: Some(event) }) => write!(formatter, "refused: {room_id} {reason} [{event}]"),
Self::Scrubbed(hit) => write!(formatter, "scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind),
Self::Trouble(why) => write!(formatter, "huginn-sources: {why}"),
}
}
}
/// A joined room the scope refused, whole, and why; `event_id` when an event did it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RefusedRoom {
@ -562,7 +595,7 @@ pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
index: &mut S,
history: &mut H,
estate: &Estate,
journal: &mut dyn FnMut(String),
journal: &mut dyn FnMut(Entry),
) -> Result<ChatRefreshed> {
let DescribedSources::Labelled { meta, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
bail!("the source collection is absent or not Huginn's; the forge refresh creates it and chat never does");
@ -589,13 +622,14 @@ pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
Ok(Err(reason)) => {
// Not kept, so the retirement below purges whatever it held.
keep.remove(room_id);
journal(format!("refused: {room_id} {reason}"));
out.refused.push(RefusedRoom { room_id: room_id.clone(), reason, event_id: None });
let refused = RefusedRoom { room_id: room_id.clone(), reason, event_id: None };
journal(Entry::Refused(refused.clone()));
out.refused.push(refused);
}
// An error is not a reclassification: the room keeps its points and cursor, and the report says why.
Err(error) => {
let report = ChatReport::failed(room_id, None, &error.context("reading the room's facts"));
journal(report.to_string());
journal(Entry::Report(report.clone()));
out.failed.push(room_id.clone());
out.rooms.push(report);
}
@ -604,35 +638,35 @@ pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
match retire_chat(index, &keep) {
Ok(done) => {
for (room, why) in done.failed {
journal(format!("huginn-sources: retiring {room} failed, the others go on: {why}"));
journal(Entry::Trouble(format!("retiring {room} failed, the others go on: {why}")));
out.failed.push(room);
}
}
Err(error) => {
journal(format!("huginn-sources: retiring rooms failed: {error:#}"));
journal(Entry::Trouble(format!("retiring rooms failed: {error:#}")));
out.failed.push("retire".to_owned());
}
}
for facts in admitted {
match ingest_room(embedder, index, history, identity.dimensions, &facts, &mut out) {
Ok(Outcome::Done(report)) => {
journal(report.to_string());
journal(Entry::Report(report.clone()));
out.rooms.push(report);
}
Ok(Outcome::Refused(refused)) => {
journal(format!("refused: {} {} [{}]", refused.room_id, refused.reason, refused.event_id.as_deref().unwrap_or("")));
journal(Entry::Refused(refused.clone()));
out.refused.push(refused);
}
Err(error) => {
let report = ChatReport::failed(&facts.room_id, facts.alias.clone(), &error);
journal(report.to_string());
journal(Entry::Report(report.clone()));
out.failed.push(facts.room_id.clone());
out.rooms.push(report);
}
}
}
for hit in &out.scrubbed {
journal(format!("scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind));
journal(Entry::Scrubbed(hit.clone()));
}
Ok(out)
}

View file

@ -141,7 +141,7 @@ impl Rig {
fn run(&mut self, history: &mut FakeHistory) -> (Result<ChatRefreshed>, Vec<String>) {
let mut journal = Vec::new();
let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line));
let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line.to_string()));
(out, journal)
}

View file

@ -322,6 +322,7 @@ mod tests {
use serde_json::json;
use super::*;
use crate::sources::test_http::{Reply, forge_of};
/// Only public, live, original, non-empty repos are the corpus; a repo the
/// forge does not say is safe is not indexed.
@ -400,60 +401,6 @@ mod tests {
assert!(HttpForge::new(&unstated).catalog().unwrap().shortfall.unwrap().contains("X-Total-Count"));
}
/// What a fake forge answers: a status, the `Location` header when there is one, and a body.
struct Reply {
status: u16,
location: Option<String>,
body: String,
}
impl From<String> for Reply {
fn from(body: String) -> Self {
Self { status: 200, location: None, body }
}
}
impl Reply {
/// What the live forge answers an anonymous reader for a private repo.
fn not_found() -> Self {
Self { status: 404, location: None, body: json!({ "message": "The target couldn't be found." }).to_string() }
}
/// What the live forge answers for a renamed repo: a 301 to its public address, `/forge` prefix and all.
fn moved_to(location: &str) -> Self {
Self { status: 301, location: Some(location.to_owned()), body: String::new() }
}
}
/// Answers each request by its path from `routes`; a path it has no answer for is held open and never answered, as a hung forge
/// does. Every path asked is recorded in order.
fn forge_of<R: Into<Reply>>(routes: impl Fn(&str) -> Option<R> + Send + 'static) -> (String, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let asked = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let record = std::sync::Arc::clone(&asked);
std::thread::spawn(move || {
let mut held = Vec::new();
for stream in listener.incoming() {
let mut stream = stream.unwrap();
let mut buffer = [0_u8; 4096];
let read = stream.read(&mut buffer).unwrap();
let request = String::from_utf8_lossy(&buffer[..read]).into_owned();
let path = request.split_whitespace().nth(1).unwrap_or_default().to_owned();
record.lock().unwrap().push(path.clone());
match routes(&path).map(Into::into) {
Some(Reply { status, location, body }) => {
let location = location.map(|location| format!("location: {location}\r\n")).unwrap_or_default();
write!(stream, "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\n{location}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap();
}
None => held.push(stream),
}
}
});
(base, asked)
}
const FEED_SAMPLE: &str = include_str!("../../tests/fixtures/feed.json");
const NOW: &str = "2026-11-03T12:00:00Z";
@ -523,7 +470,7 @@ mod tests {
assert!(read.open_prs.is_empty() && read.unseen.is_empty() && read.capped.is_empty() && read.moved.is_empty(), "a cursor asks for neither list");
let wire = serde_json::to_string(&read).unwrap();
assert!(!wire.contains("CANARY") && !wire.contains("obey") && !wire.contains("Message"), "no forge text leaves the read: {wire}");
let asked = asked.lock().unwrap().clone();
let asked = asked.paths();
assert!(!asked.is_empty() && asked.iter().all(|path| names.iter().any(|name| path.starts_with(&format!("/api/v1/repos/{name}/activities/feeds")))), "{asked:?}");
}
@ -555,13 +502,13 @@ mod tests {
let far = HttpForge::new(&base).activity(&wanted, &me, Some(5), now).unwrap();
assert!(far.capped == repos(&["GameCult/Huginn"]), "stopped at the cap with the cursor not reached");
assert_eq!((far.events.len(), far.head), (200, 2500));
assert_eq!(asked.lock().unwrap().len(), 4, "four pages and no more");
assert_eq!(asked.paths().len(), 4, "four pages and no more");
let (base, asked) = forge_of(feed("nibu"));
let near = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap();
assert!(near.capped.is_empty());
assert_eq!((near.events.len(), near.head), (20, 2500), "ids above the cursor only");
assert_eq!(asked.lock().unwrap().len(), 1);
assert_eq!(asked.paths().len(), 1);
let (base, _) = forge_of(feed("eureka"));
let hers = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap();
@ -583,7 +530,7 @@ mod tests {
let read = HttpForge::new(&base).activity(&repos(&["GameCult/Ratatoskr"]), &me(), None, at(NOW)).unwrap();
assert_eq!(read.events.iter().map(|event| event.id).collect::<Vec<_>>(), [300]);
assert!(read.capped.is_empty());
assert!(asked.lock().unwrap().iter().all(|path| !path.contains("/orgs/GameCult/activities")));
assert!(asked.paths().iter().all(|path| !path.contains("/orgs/GameCult/activities")));
}
/// A first read keeps what is inside 72 hours and nothing older, and its cursor is the newest id it saw, an entry past the window
@ -619,13 +566,13 @@ mod tests {
let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) });
let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap();
assert_eq!((read.events.len(), read.capped.len()), (53, 0));
assert_eq!(asked.lock().unwrap().iter().filter(|path| is_feed(path)).count(), 2, "page 2 was asked, page 3 was not");
assert_eq!(asked.paths().iter().filter(|path| is_feed(path)).count(), 2, "page 2 was asked, page 3 was not");
let entries: Vec<Value> = (0..400).map(|n| push_at(5000 - n, "GameCult/Huginn", "Nibu", 5)).collect();
let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) });
let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap();
assert_eq!((read.events.len(), read.capped), (REPO_FEED_PAGES * PAGE, repos(&["GameCult/Huginn"])), "200 read and the rest disclosed");
assert_eq!(asked.lock().unwrap().iter().filter(|path| is_feed(path)).count(), REPO_FEED_PAGES);
assert_eq!(asked.paths().iter().filter(|path| is_feed(path)).count(), REPO_FEED_PAGES);
}
/// With no cursor the read also lists the open PRs in her repos and hers anywhere, and the repos the forge's listing does not hold.
@ -660,14 +607,14 @@ mod tests {
assert_eq!(first.unseen, repos(&["GameCult/Ghost"]), "r60 is on page 2 of the listing");
let window = at("2026-10-31T12:00:00Z").timestamp_millis() as u64;
assert!(!first.events.is_empty() && first.events.iter().all(|event| event.at >= window), "inside 72 h: {:?}", first.events);
let paths = asked.lock().unwrap().clone();
let paths = asked.paths();
let count = |what: &str| paths.iter().filter(|path| path.contains(what)).count();
assert_eq!((count("/issues/search"), count("/orgs/GameCult/repos")), (2, 2), "a short page ends each: {paths:?}");
// A cursor asks for neither list.
let before = paths.len();
HttpForge::new(&base).activity(&wanted, &me(), Some(1), at(NOW)).unwrap();
let after = asked.lock().unwrap().clone();
let after = asked.paths();
assert!(after[before..].iter().all(|path| is_feed(path)), "only the feeds: {after:?}");
}
@ -693,12 +640,12 @@ mod tests {
});
let read = HttpForge::new(&base).activity(&repos(&["GameCult/Huginn"]), &me(), None, at(NOW)).unwrap();
assert_eq!(read.open_prs.iter().map(|pr| pr.number).collect::<Vec<_>>(), [7], "page 3 was read");
assert_eq!(asked.lock().unwrap().iter().filter(|path| path.contains("/issues/search")).count(), 4, "to the short page");
assert_eq!(asked.paths().iter().filter(|path| path.contains("/issues/search")).count(), 4, "to the short page");
}
/// The forge as it answers an anonymous reader: public repos give their feed (`[]` when quiet), a private repo answers 404, a
/// renamed repo answers 301 to a path under `/forge` that this loopback does not serve (asking it gives 404).
fn live_forge(entries: Vec<Value>, public: &'static [&'static str], listing: String) -> (String, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
fn live_forge(entries: Vec<Value>, public: &'static [&'static str], listing: String) -> (String, crate::sources::test_http::Asked) {
forge_of(move |path| {
if path.starts_with("/forge/") {
return Some(Reply::not_found());
@ -733,7 +680,7 @@ mod tests {
assert_eq!(first.unseen, repos(&["GameCult/gamecult-ops", "GameCult/cinny"]));
assert_eq!(first.moved, [RepoMoved { from: "GameCult/cinny".into(), to: "GameCult/Ratatoskr".into() }]);
assert!(first.capped.is_empty());
assert!(asked.lock().unwrap().iter().all(|path| !path.starts_with("/forge/")), "a redirect is not followed: {:?}", asked.lock().unwrap());
assert!(asked.paths().iter().all(|path| !path.starts_with("/forge/")), "a redirect is not followed: {:?}", asked.paths());
let later = forge.activity(&wanted, &me(), Some(250), at(NOW)).unwrap();
assert_eq!((later.events.iter().map(|event| event.id).collect::<Vec<_>>(), later.head), (vec![300], 300), "a later read goes on past the dark repos");

View file

@ -0,0 +1,128 @@
//! What one `huginn-sources` run does with the pieces it is handed: the forge
//! refresh, then the chat refresh (or, with chat off, the retirement of every
//! chat point), one exit status for both, and the steward-purge request. The
//! binary supplies the real forge, embedder, vector store and Matrix client and
//! owns the flags and the lock; this file decides nothing about scope or
//! scrubbing (`room_scope` and `redact` do) and never sees a credential.
use std::collections::BTreeSet;
use std::path::Path;
use anyhow::{Context, Result, bail, ensure};
use chrono::{DateTime, Utc};
use super::chat::{Entry, RoomHistory, refresh_chat, retire_chat};
use super::chat_purge::{PurgeWindow, parse_windows, purge_key, request_purge};
use super::room_scope::Estate;
use super::{DescribedSources, Forge, SourceIndex, refresh};
use crate::index::{Embedder, INDEX_CALL_TIMEOUT};
/// Chat on: the reader's rooms and the estate ids the scope refuses.
pub struct Chat<'a, H> {
pub history: &'a mut H,
pub estate: &'a Estate,
}
/// One run: the forge refresh, then chat (`Some`) or the retirement of every chat point (`None`).
/// `say` takes the journal's report lines (stdout), `complain` the failures, refusals and scrub hits (stderr).
/// A forge failure never skips the chat run and a chat failure never hides a forge one.
/// Returns the exit status: 0 only when both were clean.
pub fn run<E: Embedder, S: SourceIndex, F: Forge, H: RoomHistory>(
embedder: &mut E,
index: &mut S,
forge: &mut F,
state_root: &Path,
chat: Option<Chat<'_, H>>,
say: &mut dyn FnMut(String),
complain: &mut dyn FnMut(String),
) -> u8 {
let mut clean = true;
match refresh(embedder, index, forge, state_root, &mut |report| say(report.to_string())) {
Ok(refreshed) if refreshed.is_clean() => {}
Ok(refreshed) => {
clean = false;
if !refreshed.failed.is_empty() {
complain(format!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", ")));
}
if let Some(why) = &refreshed.retirement_withheld {
complain(format!("huginn-sources: no repo was retired: {why}"));
}
}
Err(error) => {
clean = false;
complain(format!("huginn-sources: {error:#}"));
}
}
match chat {
Some(Chat { history, estate }) => {
let mut line = |entry: Entry| if entry.is_alarm() { complain(entry.to_string()) } else { say(entry.to_string()) };
match refresh_chat(embedder, index, history, estate, &mut line) {
Ok(refreshed) => clean &= refreshed.is_clean(),
Err(error) => {
clean = false;
complain(format!("huginn-sources: chat: {error:#}"));
}
}
}
None => match retire_chat(index, &BTreeSet::new()) {
Ok(done) => {
if !done.retired.is_empty() {
say(format!("chat retired: {} rooms", done.retired.len()));
}
for (room, why) in done.failed {
clean = false;
complain(format!("huginn-sources: retiring {room} failed: {why}"));
}
}
Err(error) => {
clean = false;
complain(format!("huginn-sources: chat: {error:#}"));
}
},
}
u8::from(!clean)
}
/// The longest span one request may name.
const PURGE_SPAN_DAYS: i64 = 366;
const REQUEST_BYTES_MAX: usize = 120;
/// A steward's purge request as the admin typed it, checked. Each error names its option.
#[derive(Debug, PartialEq, Eq)]
pub struct Purge {
pub room_id: String,
pub from: String,
pub to: String,
pub request: String,
pub window: PurgeWindow,
}
fn instant(option: &str, text: &str) -> Result<DateTime<Utc>> {
ensure!(text.ends_with('Z'), "--{option} must be an RFC 3339 UTC time ending in Z, got {text:?}");
let at = DateTime::parse_from_rfc3339(text).map_err(|_| anyhow::anyhow!("--{option} must be an RFC 3339 UTC time ending in Z, got {text:?}"))?;
Ok(at.with_timezone(&Utc))
}
impl Purge {
pub fn check(room: &str, from: &str, to: &str, request: &str, now: DateTime<Utc>) -> Result<Self> {
ensure!(room.starts_with('!'), "--room must be a room id beginning with !, got {room:?}: an alias needs Matrix, and the journal prints both #alias (!id)");
let (start, end) = (instant("from", from)?, instant("to", to)?);
ensure!(start < end, "--from must be before --to");
ensure!(end <= now, "--to must not be after now");
ensure!(end - start <= chrono::Duration::days(PURGE_SPAN_DAYS), "the window must be at most {PURGE_SPAN_DAYS} days");
ensure!(!request.is_empty() && request.len() <= REQUEST_BYTES_MAX, "--request must be 1 to {REQUEST_BYTES_MAX} bytes");
let ms = |at: DateTime<Utc>| u64::try_from(at.timestamp_millis()).context("a purge time before 1970");
Ok(Self { room_id: room.to_owned(), from: from.to_owned(), to: to.to_owned(), request: request.to_owned(), window: PurgeWindow { from_ms: ms(start)?, to_ms: ms(end)? } })
}
}
/// Writes the tombstone and returns the line the admin sees; the free-text request is quoted and escaped so it cannot forge a journal line. Touches no point and makes no Matrix call;
/// the caller holds the job's lock. The next run applies it.
pub fn purge<S: SourceIndex>(index: &mut S, purge: &Purge) -> Result<String> {
request_purge(index, &purge.room_id, purge.window)?;
let DescribedSources::Labelled { cursors, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
bail!("the source collection is absent or not Huginn's");
};
let windows = cursors.get(&purge_key(&purge.room_id)).map(|held| parse_windows(held)).transpose()?.map_or(0, |held| held.len());
Ok(format!("chat-purge requested: {} {}..{} request={:?} windows={windows}", purge.room_id, purge.from, purge.to, purge.request))
}

View file

@ -0,0 +1,287 @@
//! One run of the job: forge refresh and chat refresh over a fake Synapse reached through the real client,
//! a fake vector store and the fake embedder; the exit status; the chat-off retirement; the purge request.
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use anyhow::{Result, bail};
use chrono::{DateTime, Utc};
use serde_json::json;
use super::chat::{CHAT_PURGE_PREFIX, RoomHistory};
use super::job::{Chat, Purge, purge, run};
use super::matrix::HttpMatrix;
use super::matrix_tests::{Room, Rooms, TOKEN, message, state, synapse};
use super::room_scope::Estate;
use super::tests::{FakeForge, Remote};
use super::test_http::forge_of;
use super::*;
use crate::index::fakes::{FakeEmbedder, FakeSources};
const DAY0: u64 = 1_794_268_800_000; // 2026-11-10T00:00:00Z
fn now() -> DateTime<Utc> {
DateTime::parse_from_rfc3339("2026-11-17T00:00:00Z").unwrap().with_timezone(&Utc)
}
fn estate() -> Estate {
Estate::new(["!e1:s", "!e2:s", "!e3:s"].map(str::to_owned).into()).unwrap()
}
struct Out {
said: Vec<String>,
complained: Vec<String>,
status: u8,
}
struct Rig {
embedder: FakeEmbedder,
index: FakeSources,
forge: FakeForge,
_remote: Remote,
root: tempfile::TempDir,
}
impl Rig {
fn new() -> Self {
let remote = Remote::new();
remote.commit(&[("README.md", b"# Demo\n")], &[]);
Self { embedder: FakeEmbedder::new("d1"), index: FakeSources::default(), forge: FakeForge::of(&[("Demo", &remote)]), _remote: remote, root: tempfile::tempdir().unwrap() }
}
fn go<H: RoomHistory>(&mut self, history: Option<&mut H>) -> Out {
let (mut said, mut complained) = (Vec::new(), Vec::new());
let estate = estate();
let chat = history.map(|history| Chat { history, estate: &estate });
let status = run(&mut self.embedder, &mut self.index, &mut self.forge, self.root.path(), chat, &mut |line| said.push(line), &mut |line| complained.push(line));
Out { said, complained, status }
}
fn chat_bodies(&self) -> Vec<String> {
let mut bodies: Vec<String> = self.index.state.lock().unwrap().chat.values().flat_map(|point| point.payload.messages.iter().map(|m| m.body.clone())).collect();
bodies.sort();
bodies
}
fn chat_rooms(&self) -> std::collections::BTreeSet<String> {
self.index.state.lock().unwrap().chat.values().map(|point| point.payload.room_id.clone()).collect()
}
}
fn rooms(list: Vec<(&str, Vec<serde_json::Value>, Vec<serde_json::Value>)>) -> Rooms {
Arc::new(Mutex::new(list.into_iter().map(|(id, state, timeline)| (id.to_owned(), Room { state, timeline })).collect::<BTreeMap<_, _>>()))
}
fn platform() -> String {
["disc", "ord"].concat()
}
fn world() -> Rooms {
let relay = format!("@_{}_1:s", platform());
rooms(vec![
("!swarm:s", vec![state("m.room.canonical_alias", json!({ "alias": "#swarm:gamecult.org" })), state("m.room.join_rules", json!({ "join_rule": "invite" }))], vec![
message("$m1", "@a:s", DAY0 - 5000, "first words"),
message("$m2", "@b:s", DAY0 - 4000, "to be redacted"),
message("$m3", "@a:s", DAY0 + 1000, "inside the window"),
]),
("!bridged:s", vec![state("m.bridge", json!({}))], vec![message("$b1", "@a:s", DAY0, "bridged talk")]),
("!relay:s", vec![], vec![message("$d1", &relay, DAY0, "relayed talk")]),
("!aliased:s", vec![state("m.room.canonical_alias", json!({ "alias": "#Users:gamecult.org" }))], vec![message("$u1", "@a:s", DAY0, "estate talk")]),
])
}
fn all_lines(out: &Out) -> String {
out.said.iter().chain(&out.complained).cloned().collect::<Vec<_>>().join("\n")
}
#[test]
fn a_full_run_over_a_fake_synapse() {
let world = world();
let (base, asked) = synapse(&world);
let mut matrix = HttpMatrix::new(&base, TOKEN);
let mut rig = Rig::new();
let first = rig.go(Some(&mut matrix));
assert_eq!(first.status, 1, "joined bridged, relay and estate rooms are refused: {:?}", first.complained);
assert_eq!(rig.chat_rooms(), ["!swarm:s".to_owned()].into(), "an invite-only room is ingested; nothing of the refused rooms is");
assert_eq!(rig.chat_bodies(), ["first words", "inside the window", "to be redacted"]);
for refused in ["!bridged:s", "!relay:s", "!aliased:s"] {
assert!(first.complained.iter().any(|line| line.starts_with("refused:") && line.contains(refused)), "{refused}: {:?}", first.complained);
}
assert!(!rig.embedder.texts().iter().any(|text| text.contains("bridged talk") || text.contains("relayed talk") || text.contains("estate talk")));
assert!(first.said.iter().any(|line| line.starts_with("Demo ")), "the forge's line is journaled too");
assert!(first.said.iter().any(|line| line.starts_with("#swarm:gamecult.org (!swarm:s)") && line.contains("events=3")), "{:?}", first.said);
assert!(asked.seen().iter().all(|seen| seen.method == "GET"));
// The steward asks for the window; then the room edits, redacts and leaks a key.
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "follow_up:x", now()).unwrap();
assert_eq!(purge(&mut rig.index, &request).unwrap(), "chat-purge requested: !swarm:s 2026-11-10T00:00:00Z..2026-11-11T00:00:00Z request=\"follow_up:x\" windows=1");
let before = rig.index.cursors();
assert_eq!(purge(&mut rig.index, &request).unwrap().ends_with("windows=1"), true);
assert_eq!(rig.index.cursors(), before, "a repeated request writes nothing");
let secret = format!("key ghp_{}", "k2L9".repeat(9));
{
let mut held = world.lock().unwrap();
let room = held.get_mut("!swarm:s").unwrap();
room.timeline.push(json!({ "type": "m.room.message", "event_id": "$edit", "sender": "@a:s", "origin_server_ts": DAY0 + 90_000_000, "content": {
"msgtype": "m.text", "body": "* first words, amended", "m.new_content": { "msgtype": "m.text", "body": "first words, amended" }, "m.relates_to": { "rel_type": "m.replace", "event_id": "$m1" } } }));
room.timeline.push(json!({ "type": "m.room.redaction", "event_id": "$red", "sender": "@b:s", "origin_server_ts": DAY0 + 91_000_000, "content": { "redacts": "$m2" } }));
room.timeline.push(message("$m4", "@a:s", DAY0 + 92_000_000, &secret));
}
let second = rig.go(Some(&mut matrix));
assert_eq!(rig.chat_bodies().len(), 2, "{:?}", rig.chat_bodies());
let bodies = rig.chat_bodies().join("\n");
assert!(bodies.contains("first words, amended") && !bodies.contains("to be redacted") && !bodies.contains("inside the window"), "{bodies}");
assert!(!bodies.contains("ghp_k2L9") && !rig.embedder.texts().iter().any(|text| text.contains("ghp_k2L9")), "the key was scrubbed before it was embedded");
let text = all_lines(&second);
assert!(second.said.iter().any(|line| line.contains("purged=")), "{text}");
assert!(second.complained.iter().any(|line| line == "scrubbed: !swarm:s $m4 key-shape" || line.starts_with("scrubbed: !swarm:s $m4 ")), "{text}");
for body in ["first words", "to be redacted", "inside the window", "relayed talk", "ghp_"] {
assert!(!text.contains(body), "the journal carries ids and counts, not bodies: {body}");
}
let third = rig.go(Some(&mut matrix));
assert!(!rig.chat_bodies().join("\n").contains("inside the window"), "a purged window does not come back");
assert!(third.complained.iter().all(|line| line.starts_with("refused:")), "{:?}", third.complained);
}
#[test]
fn no_chat_flags_purge_the_corpus_and_make_no_matrix_call() {
let world = world();
let (base, _) = synapse(&world);
let mut matrix = HttpMatrix::new(&base, TOKEN);
let mut rig = Rig::new();
rig.go(Some(&mut matrix));
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "r", now()).unwrap();
purge(&mut rig.index, &request).unwrap();
assert!(!rig.chat_bodies().is_empty());
let (hung, asked) = forge_of(|_| None::<String>);
drop(hung);
let out = rig.go::<HttpMatrix>(None);
assert_eq!(out.status, 0, "{:?}", out.complained);
assert!(rig.chat_bodies().is_empty() && rig.chat_rooms().is_empty());
assert!(rig.index.cursors().keys().any(|key| key.starts_with(CHAT_PURGE_PREFIX)), "tombstones stay");
assert!(!rig.index.cursors().keys().any(|key| key.starts_with("chat:")), "cursors go");
assert!(asked.seen().is_empty());
}
struct BrokenForge;
impl Forge for BrokenForge {
fn catalog(&mut self) -> Result<Catalog> {
bail!("the forge is down")
}
fn clone_url(&self, _name: &str) -> String {
String::new()
}
}
struct DeadSynapse;
impl RoomHistory for DeadSynapse {
fn joined_rooms(&mut self) -> Result<Vec<String>> {
bail!("synapse is down")
}
fn facts(&mut self, _room_id: &str) -> Result<super::room_scope::RoomFacts> {
bail!("synapse is down")
}
fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result<Option<super::chat::EventPage>> {
bail!("synapse is down")
}
}
#[test]
fn exit_status_is_clean_only_when_forge_and_chat_both_are() {
let clean = rooms(vec![("!swarm:s", vec![], vec![message("$m1", "@a:s", 1, "words")])]);
let (base, _) = synapse(&clean);
let mut rig = Rig::new();
let out = rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
assert_eq!((out.status, out.complained.len()), (0, 0));
// Scrub hits alone are not a failure; they are reported.
clean.lock().unwrap().get_mut("!swarm:s").unwrap().timeline.push(message("$m2", "@a:s", 2, &format!("ghp_{}", "k2L9".repeat(9))));
let out = rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
assert_eq!(out.status, 0, "{:?}", out.complained);
assert!(out.complained.iter().any(|line| line.starts_with("scrubbed: !swarm:s $m2")));
// A chat failure still reports the forge, and a forge failure still runs chat.
let out = rig.go(Some(&mut DeadSynapse));
assert_eq!(out.status, 1);
assert!(out.complained.iter().any(|line| line.contains("synapse is down")));
let (mut said, mut complained) = (Vec::new(), Vec::new());
let (estate, mut ok) = (estate(), HttpMatrix::new(&base, TOKEN));
let status = run(&mut rig.embedder, &mut rig.index, &mut BrokenForge, rig.root.path(), Some(Chat { history: &mut ok, estate: &estate }), &mut |l| said.push(l), &mut |l| complained.push(l));
assert_eq!(status, 1);
assert!(complained.iter().any(|line| line.contains("the forge is down")), "{complained:?}");
assert!(said.iter().any(|line| line.contains("(!swarm:s)")), "chat ran after the forge failed: {said:?}");
}
#[test]
fn a_request_cannot_forge_a_journal_line() {
let mut rig = Rig::new();
rig.go::<HttpMatrix>(None);
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "x\nrefused: !forged:s", now()).unwrap();
let line = purge(&mut rig.index, &request).unwrap();
assert_eq!(line.lines().count(), 1, "{line:?}");
}
#[test]
fn a_failed_room_report_goes_to_the_complaint_stream_and_fails_the_run() {
let mut rig = Rig::new();
struct FactsDown(Rooms);
impl RoomHistory for FactsDown {
fn joined_rooms(&mut self) -> Result<Vec<String>> {
Ok(vec!["!swarm:s".to_owned()])
}
fn facts(&mut self, _room_id: &str) -> Result<super::room_scope::RoomFacts> {
bail!("facts are down")
}
fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result<Option<super::chat::EventPage>> {
bail!("unreachable")
}
}
let out = rig.go(Some(&mut FactsDown(rooms(vec![]))));
assert_eq!(out.status, 1);
assert!(out.complained.iter().any(|line| line.contains("!swarm:s") && line.contains("facts are down")), "{:?}", out.complained);
assert!(!out.said.iter().any(|line| line.contains("facts are down")), "{:?}", out.said);
}
#[test]
fn chat_purge_makes_no_matrix_request() {
let world = world();
let (base, asked) = synapse(&world);
let mut rig = Rig::new();
rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
let made = asked.seen().len();
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "r", now()).unwrap();
purge(&mut rig.index, &request).unwrap();
assert_eq!(asked.seen().len(), made);
}
#[test]
fn a_retirement_failure_goes_to_the_complaint_stream() {
struct Left;
impl RoomHistory for Left {
fn joined_rooms(&mut self) -> Result<Vec<String>> {
Ok(vec![])
}
fn facts(&mut self, _room_id: &str) -> Result<super::room_scope::RoomFacts> {
bail!("unreachable")
}
fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result<Option<super::chat::EventPage>> {
bail!("unreachable")
}
}
let world = rooms(vec![("!swarm:s", vec![], vec![message("$m1", "@a:s", 1, "words")])]);
let (base, _) = synapse(&world);
let mut rig = Rig::new();
rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
rig.index.state.lock().unwrap().fail_chat_deletes = true;
let out = rig.go(Some(&mut Left));
assert_eq!(out.status, 1);
assert!(out.complained.iter().any(|line| line.starts_with("huginn-sources: retiring !swarm:s failed")), "{:?}", out.complained);
assert!(!out.said.iter().any(|line| line.contains("retiring")), "{:?}", out.said);
}

View file

@ -0,0 +1,159 @@
//! Synapse's client API as the reader user sees it: Matrix JSON in, `RoomEvent`
//! and `RoomFacts` out. JSON for Matrix is spoken here and nowhere else. This
//! client reads with GET only and holds no policy: it filters nothing, decides
//! no room's or sender's scope, and names no platform. What is in scope is
//! `room_scope`'s; the token is read at one place, the `Authorization` header.
use std::time::Duration;
use anyhow::{Context, Result, bail};
use serde_json::Value;
use super::chat::{EventKind, EventPage, RoomEvent, RoomHistory};
use super::room_scope::RoomFacts;
const CALL_TIMEOUT: Duration = Duration::from_secs(60);
/// A bridge's state event types: either marks a room as relaying another platform.
const BRIDGE_TYPES: [&str; 2] = ["m.bridge", "uk.half-shot.bridge"];
pub struct HttpMatrix {
agent: ureq::Agent,
base: String,
token: String,
}
impl std::fmt::Debug for HttpMatrix {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.debug_struct("HttpMatrix").field("base", &self.base).finish_non_exhaustive()
}
}
/// A path segment with everything but unreserved characters escaped.
fn segment(raw: &str) -> String {
let mut out = String::with_capacity(raw.len());
for byte in raw.bytes() {
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
out.push(char::from(byte));
} else {
out.push_str(&format!("%{byte:02X}"));
}
}
out
}
impl HttpMatrix {
pub fn new(base_url: &str, token: &str) -> Self {
let agent: ureq::Agent =
ureq::Agent::config_builder().timeout_global(Some(CALL_TIMEOUT)).http_status_as_error(false).build().into();
Self { agent, base: base_url.trim_end_matches('/').to_owned(), token: token.to_owned() }
}
/// One GET of `path` (already escaped): the status and the JSON body. No message made here carries the token.
fn get(&self, path: &str) -> Result<(u16, Value)> {
let url = format!("{}{path}", self.base);
let mut response = self
.agent
.get(&url)
.header("Authorization", format!("Bearer {}", self.token))
.call()
.with_context(|| format!("no answer from {url}"))?;
let status = response.status().as_u16();
let body = response.body_mut().read_to_string().with_context(|| format!("reading the answer from {url}"))?;
Ok((status, serde_json::from_str(&body).unwrap_or(Value::Null)))
}
fn ok(&self, path: &str) -> Result<Value> {
let (status, body) = self.get(path)?;
if status != 200 {
bail!("synapse answered {status} for {path}");
}
Ok(body)
}
}
fn event_of(raw: &Value) -> Result<RoomEvent> {
let text = |key: &str| raw[key].as_str().map(str::to_owned).with_context(|| format!("an event has no {key}"));
let content = &raw["content"];
let content_keys = content.as_object().map(|map| map.keys().cloned().collect()).unwrap_or_default();
Ok(RoomEvent { event_id: text("event_id")?, sender: text("sender")?, ts_ms: raw["origin_server_ts"].as_u64().context("an event has no origin_server_ts")?, content_keys, kind: kind_of(raw) })
}
fn kind_of(raw: &Value) -> EventKind {
let content = &raw["content"];
if !raw["unsigned"]["redacted_because"].is_null() {
return EventKind::Other;
}
match raw["type"].as_str() {
Some("m.room.redaction") => match content["redacts"].as_str().or_else(|| raw["redacts"].as_str()) {
Some(target) => EventKind::Redaction { target: target.to_owned() },
None => EventKind::Other,
},
Some("m.room.message") if matches!(content["msgtype"].as_str(), Some("m.text" | "m.notice" | "m.emote")) => {
let Some(body) = content["body"].as_str() else { return EventKind::Other };
let relation = &content["m.relates_to"];
if relation["rel_type"].as_str() == Some("m.replace") {
return match (relation["event_id"].as_str(), content["m.new_content"]["body"].as_str()) {
(Some(target), Some(new)) => EventKind::Edit { target: target.to_owned(), body: new.to_owned() },
_ => EventKind::Other,
};
}
EventKind::Message { body: body.to_owned() }
}
_ => EventKind::Other,
}
}
impl RoomHistory for HttpMatrix {
fn joined_rooms(&mut self) -> Result<Vec<String>> {
let body = self.ok("/_matrix/client/v3/joined_rooms")?;
let rooms = body["joined_rooms"].as_array().context("the joined rooms answer is not a list")?;
Ok(rooms.iter().filter_map(|room| room.as_str().map(str::to_owned)).collect())
}
fn facts(&mut self, room_id: &str) -> Result<RoomFacts> {
let body = self.ok(&format!("/_matrix/client/v3/rooms/{}/state", segment(room_id)))?;
let state = body.as_array().context("the room state is not a list")?;
let mut facts = RoomFacts { room_id: room_id.to_owned(), alias: None, alt_aliases: Vec::new(), name: None, encrypted: false, direct: false, bridge: None };
for event in state {
let content = &event["content"];
match event["type"].as_str() {
Some("m.room.canonical_alias") => {
facts.alias = content["alias"].as_str().map(str::to_owned);
facts.alt_aliases = content["alt_aliases"].as_array().map(|list| list.iter().filter_map(|alias| alias.as_str().map(str::to_owned)).collect()).unwrap_or_default();
}
Some("m.room.name") => facts.name = content["name"].as_str().map(str::to_owned),
Some("m.room.encryption") => facts.encrypted = true,
Some("m.room.member") if content["is_direct"].as_bool() == Some(true) => facts.direct = true,
Some(kind) if BRIDGE_TYPES.contains(&kind) => {
facts.bridge.get_or_insert_with(|| kind.to_owned());
}
_ => {}
}
}
Ok(facts)
}
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>> {
let room = segment(room_id);
let from = match after {
None => String::new(),
Some(event) => {
let (status, body) = self.get(&format!("/_matrix/client/v3/rooms/{room}/context/{}?limit=0", segment(event)))?;
match status {
200 => match body["end"].as_str() {
Some(end) => format!("&from={}", segment(end)),
None => bail!("synapse gave no end token for the context of {event} in {room_id}"),
},
404 => return Ok(None),
other => bail!("synapse answered {other} for the context of {event} in {room_id}"),
}
}
};
let body = self.ok(&format!("/_matrix/client/v3/rooms/{room}/messages?dir=f&limit={limit}{from}"))?;
let chunk = body["chunk"].as_array().context("the messages answer has no chunk")?;
let events = chunk.iter().map(event_of).collect::<Result<Vec<_>>>()?;
let more = body["end"].as_str().is_some() && !events.is_empty();
Ok(Some(EventPage { events, more }))
}
}

View file

@ -0,0 +1,207 @@
//! The Matrix client against a fake Synapse that speaks the client API's JSON.
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex};
use serde_json::{Value, json};
use super::chat::{EventKind, RoomHistory};
use super::matrix::HttpMatrix;
use super::test_http::{Asked, Reply, forge_of};
pub(super) const TOKEN: &str = "syt_reader_9f8e7d6c5b4a";
/// A room as the fake server holds it: its state events and its timeline.
pub(super) struct Room {
pub state: Vec<Value>,
pub timeline: Vec<Value>,
}
pub(super) type Rooms = Arc<Mutex<BTreeMap<String, Room>>>;
pub(super) fn decode(raw: &str) -> String {
let bytes = raw.as_bytes();
let mut out = Vec::new();
let mut at = 0;
while at < bytes.len() {
if bytes[at] == b'%' {
out.push(u8::from_str_radix(&raw[at + 1..at + 3], 16).unwrap());
at += 3;
} else {
out.push(bytes[at]);
at += 1;
}
}
String::from_utf8(out).unwrap()
}
pub(super) fn message(id: &str, sender: &str, ts: u64, body: &str) -> Value {
json!({ "type": "m.room.message", "event_id": id, "sender": sender, "origin_server_ts": ts, "content": { "msgtype": "m.text", "body": body } })
}
pub(super) fn state(kind: &str, content: Value) -> Value {
json!({ "type": kind, "state_key": "", "content": content })
}
/// A Synapse over `rooms`, joined to the rooms in the map: GET only, bearer required.
pub(super) fn synapse(rooms: &Rooms) -> (String, Asked) {
let rooms = Arc::clone(rooms);
forge_of(move |raw| {
let (path, query) = raw.split_once('?').unwrap_or((raw, ""));
let path = decode(path);
let rooms = rooms.lock().unwrap();
let rest = path.strip_prefix("/_matrix/client/v3/")?;
if rest == "joined_rooms" {
return Some(Reply::from(json!({ "joined_rooms": rooms.keys().collect::<Vec<_>>() }).to_string()));
}
let rest = rest.strip_prefix("rooms/")?;
let (id, rest) = rest.split_once('/')?;
let Some(room) = rooms.get(id) else { return Some(Reply::status(404, r#"{"errcode":"M_NOT_FOUND"}"#)) };
let query: BTreeMap<String, String> = query.split('&').filter_map(|pair| pair.split_once('=')).map(|(k, v)| (k.to_owned(), decode(v))).collect();
if rest == "state" {
return Some(Reply::from(Value::Array(room.state.clone()).to_string()));
}
if let Some(event) = rest.strip_prefix("context/") {
return Some(match room.timeline.iter().any(|held| held["event_id"] == event) {
true => Reply::from(json!({ "end": format!("t_{event}"), "events_before": [], "events_after": [] }).to_string()),
false => Reply::status(404, r#"{"errcode":"M_NOT_FOUND"}"#),
});
}
if rest == "messages" {
let start = match query.get("from") {
Some(from) => 1 + room.timeline.iter().position(|held| format!("t_{}", held["event_id"].as_str().unwrap()) == *from)?,
None => 0,
};
let limit: usize = query["limit"].parse().unwrap();
let chunk: Vec<&Value> = room.timeline.iter().skip(start).take(limit).collect();
let end = chunk.last().map(|last| format!("t_{}", last["event_id"].as_str().unwrap()));
return Some(Reply::from(json!({ "chunk": chunk, "end": end }).to_string()));
}
None
})
}
fn one_room(state: Vec<Value>, timeline: Vec<Value>) -> Rooms {
Arc::new(Mutex::new(BTreeMap::from([("!r:s".to_owned(), Room { state, timeline })])))
}
#[test]
fn state_maps_to_facts_and_the_join_rule_is_not_read() {
let plain = one_room(vec![state("m.room.join_rules", json!({ "join_rule": "invite" }))], vec![]);
let (base, _) = synapse(&plain);
let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap();
assert_eq!((facts.alias, facts.name, facts.encrypted, facts.direct, facts.bridge), (None, None, false, false, None));
for rule in ["public", "knock", "invite"] {
let rooms = one_room(vec![state("m.room.join_rules", json!({ "join_rule": rule }))], vec![]);
let (base, _) = synapse(&rooms);
let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap();
assert!(!facts.direct && !facts.encrypted && facts.bridge.is_none(), "{rule}");
}
let full = one_room(
vec![
state("m.room.canonical_alias", json!({ "alias": "#a:s", "alt_aliases": ["#b:s", "#c:s"] })),
state("m.room.name", json!({ "name": "Swarm" })),
state("m.room.encryption", json!({ "algorithm": "m.megolm.v1.aes-sha2" })),
json!({ "type": "m.room.member", "state_key": "@x:s", "content": { "membership": "join" } }),
json!({ "type": "m.room.member", "state_key": "@y:s", "content": { "membership": "join", "is_direct": true } }),
state("uk.half-shot.bridge", json!({})),
],
vec![],
);
let (base, _) = synapse(&full);
let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap();
assert_eq!(facts.alias.as_deref(), Some("#a:s"));
assert_eq!(facts.alt_aliases, ["#b:s", "#c:s"]);
assert_eq!(facts.name.as_deref(), Some("Swarm"));
assert!(facts.encrypted && facts.direct);
assert_eq!(facts.bridge.as_deref(), Some("uk.half-shot.bridge"));
let relay = one_room(vec![state("m.bridge", json!({}))], vec![]);
let (base, _) = synapse(&relay);
assert_eq!(HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap().bridge.as_deref(), Some("m.bridge"));
}
#[test]
fn events_map_to_kinds_and_content_keys_are_names_only() {
let mut other = message("$img", "@a:s", 3, "pic.png");
other["content"]["msgtype"] = json!("m.image");
let mut pruned = message("$pruned", "@a:s", 4, "was here");
pruned["unsigned"] = json!({ "redacted_because": { "event_id": "$x" } });
let mut notice = message("$n", "@a:s", 5, "n");
notice["content"]["msgtype"] = json!("m.notice");
let mut emote = message("$e", "@a:s", 6, "e");
emote["content"]["msgtype"] = json!("m.emote");
let edit = json!({ "type": "m.room.message", "event_id": "$edit", "sender": "@a:s", "origin_server_ts": 7, "content": {
"msgtype": "m.text", "body": "* new", "m.new_content": { "msgtype": "m.text", "body": "new" }, "m.relates_to": { "rel_type": "m.replace", "event_id": "$m1" } } });
let redaction = json!({ "type": "m.room.redaction", "event_id": "$red", "sender": "@a:s", "origin_server_ts": 8, "content": { "redacts": "$m1" } });
let old_redaction = json!({ "type": "m.room.redaction", "event_id": "$red2", "sender": "@a:s", "origin_server_ts": 9, "redacts": "$m2", "content": {} });
let encrypted = json!({ "type": "m.room.encrypted", "event_id": "$enc", "sender": "@a:s", "origin_server_ts": 10, "content": { "ciphertext": "xx" } });
let timeline = vec![message("$m1", "@a:s", 1, "hello"), other, pruned, notice, emote, edit, redaction, old_redaction, encrypted];
let rooms = one_room(vec![], timeline);
let (base, _) = synapse(&rooms);
let page = HttpMatrix::new(&base, TOKEN).events_after("!r:s", None, 50).unwrap().unwrap();
let kinds: Vec<EventKind> = page.events.iter().map(|event| event.kind.clone()).collect();
assert_eq!(
kinds,
[
EventKind::Message { body: "hello".into() },
EventKind::Other,
EventKind::Other,
EventKind::Message { body: "n".into() },
EventKind::Message { body: "e".into() },
EventKind::Edit { target: "$m1".into(), body: "new".into() },
EventKind::Redaction { target: "$m1".into() },
EventKind::Redaction { target: "$m2".into() },
EventKind::Other,
]
);
assert_eq!(page.events[0].content_keys, ["body", "msgtype"]);
assert_eq!(page.events[5].content_keys, ["body", "m.new_content", "m.relates_to", "msgtype"], "names, never values");
assert_eq!((page.events[0].event_id.as_str(), page.events[0].sender.as_str(), page.events[0].ts_ms), ("$m1", "@a:s", 1));
}
#[test]
fn resume_uses_the_cursor_events_end_token_and_a_missing_event_is_none() {
let rooms = one_room(vec![], vec![message("$1", "@a:s", 1, "a"), message("$2", "@a:s", 2, "b"), message("$3", "@a:s", 3, "c")]);
let (base, asked) = synapse(&rooms);
let mut matrix = HttpMatrix::new(&base, TOKEN);
let page = matrix.events_after("!r:s", Some("$1"), 1).unwrap().unwrap();
assert_eq!(page.events.iter().map(|event| event.event_id.as_str()).collect::<Vec<_>>(), ["$2"]);
assert!(page.more);
assert_eq!(asked.paths(), ["/_matrix/client/v3/rooms/%21r%3As/context/%241?limit=0", "/_matrix/client/v3/rooms/%21r%3As/messages?dir=f&limit=1&from=t_%241"]);
assert!(matrix.events_after("!r:s", Some("$unknown"), 5).unwrap().is_none());
let tail = matrix.events_after("!r:s", Some("$3"), 5).unwrap().unwrap();
assert!(tail.events.is_empty() && !tail.more, "an empty chunk ends the run");
let first = matrix.events_after("!r:s", None, 2).unwrap().unwrap();
assert_eq!(first.events.len(), 2);
assert_eq!(asked.paths().last().unwrap(), "/_matrix/client/v3/rooms/%21r%3As/messages?dir=f&limit=2", "no from, no context call");
}
#[test]
fn only_get_always_the_bearer_and_the_token_is_never_printed() {
let rooms = one_room(vec![], vec![message("$1", "@a:s", 1, "a")]);
let (base, asked) = synapse(&rooms);
let mut matrix = HttpMatrix::new(&base, TOKEN);
matrix.joined_rooms().unwrap();
matrix.facts("!r:s").unwrap();
matrix.events_after("!r:s", Some("$1"), 3).unwrap();
let seen = asked.seen();
assert!(seen.len() >= 4 && seen.iter().all(|seen| seen.method == "GET" && seen.authorization.as_deref() == Some(&format!("Bearer {TOKEN}"))), "{seen:?}");
let (base, _) = forge_of(|_| Some(Reply::status(401, &format!(r#"{{"errcode":"M_UNKNOWN_TOKEN","error":"bad {TOKEN}"}}"#))));
let mut matrix = HttpMatrix::new(&base, TOKEN);
let error = matrix.joined_rooms().unwrap_err();
assert!(format!("{error:#}").contains("401"));
assert!(!format!("{error:#}{error:?}").contains(TOKEN));
assert!(!format!("{matrix:?}").contains(TOKEN));
}
#[test]
fn ids_are_percent_encoded_in_paths() {
let weird = "$a+b/c=d:s";
let rooms: Rooms = Arc::new(Mutex::new(BTreeMap::from([("!ro om:s".to_owned(), Room { state: vec![], timeline: vec![message(weird, "@a:s", 1, "x"), message("$2", "@a:s", 2, "y")] })])));
let (base, asked) = synapse(&rooms);
let mut matrix = HttpMatrix::new(&base, TOKEN);
let page = matrix.events_after("!ro om:s", Some(weird), 5).unwrap().unwrap();
assert_eq!(page.events.len(), 1, "the server decoded the room and event ids to the originals");
let raw = asked.paths();
assert!(raw.iter().all(|path| !path.contains('+') && !path.contains(' ') && !path.contains("!ro")), "{raw:?}");
}

View file

@ -30,11 +30,20 @@ pub mod chat_purge;
pub mod chunk;
pub mod forge;
pub mod git;
pub mod job;
pub mod matrix;
pub mod room_scope;
#[cfg(test)]
mod test_http;
#[cfg(test)]
mod chat_tests;
#[cfg(test)]
mod job_tests;
#[cfg(test)]
mod matrix_tests;
#[cfg(test)]
mod tests;
use std::collections::hash_map::Entry;

View file

@ -0,0 +1,92 @@
//! The one fake HTTP server the source tests talk to (the forge's and Matrix's):
//! it answers by path, holds open a path it has no answer for as a hung server
//! does, and records each request's method, path and `Authorization` header.
use std::io::{Read, Write};
use std::sync::{Arc, Mutex};
use serde_json::json;
/// What a fake server answers: a status, the `Location` header when there is one, and a body.
pub struct Reply {
pub status: u16,
pub location: Option<String>,
pub body: String,
}
impl From<String> for Reply {
fn from(body: String) -> Self {
Self { status: 200, location: None, body }
}
}
impl Reply {
/// What the live forge answers an anonymous reader for a private repo.
pub fn not_found() -> Self {
Self { status: 404, location: None, body: json!({ "message": "The target couldn't be found." }).to_string() }
}
/// What the live forge answers for a renamed repo: a 301 to its public address, `/forge` prefix and all.
pub fn moved_to(location: &str) -> Self {
Self { status: 301, location: Some(location.to_owned()), body: String::new() }
}
pub fn status(status: u16, body: &str) -> Self {
Self { status, location: None, body: body.to_owned() }
}
}
/// One request as the server saw it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Seen {
pub method: String,
pub path: String,
pub authorization: Option<String>,
}
/// Every request a fake server has seen, in order.
#[derive(Clone, Default)]
pub struct Asked(Arc<Mutex<Vec<Seen>>>);
impl Asked {
pub fn seen(&self) -> Vec<Seen> {
self.0.lock().unwrap().clone()
}
pub fn paths(&self) -> Vec<String> {
self.0.lock().unwrap().iter().map(|seen| seen.path.clone()).collect()
}
}
/// Answers each request by its path from `routes`; a path it has no answer for is held open and never answered.
pub fn forge_of<R: Into<Reply>>(routes: impl Fn(&str) -> Option<R> + Send + 'static) -> (String, Asked) {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let asked = Asked::default();
let record = asked.clone();
std::thread::spawn(move || {
let mut held = Vec::new();
for stream in listener.incoming() {
let mut stream = stream.unwrap();
let mut buffer = [0_u8; 4096];
let read = stream.read(&mut buffer).unwrap();
let request = String::from_utf8_lossy(&buffer[..read]).into_owned();
let mut words = request.split_whitespace();
let method = words.next().unwrap_or_default().to_owned();
let path = words.next().unwrap_or_default().to_owned();
let authorization = request
.lines()
.find_map(|line| line.split_once(':').filter(|(name, _)| name.eq_ignore_ascii_case("authorization")))
.map(|(_, value)| value.trim().to_owned());
record.0.lock().unwrap().push(Seen { method, path: path.clone(), authorization });
match routes(&path).map(Into::into) {
Some(Reply { status, location, body }) => {
let location = location.map(|location| format!("location: {location}\r\n")).unwrap_or_default();
write!(stream, "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\n{location}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap();
}
None => held.push(stream),
}
}
});
(base, asked)
}

View file

@ -25,23 +25,23 @@ fn git_in(dir: &Path, args: &[&str]) -> String {
}
/// A working repo on disk, committed to by the test.
struct Remote {
pub(super) struct Remote {
dir: tempfile::TempDir,
}
impl Remote {
fn new() -> Self {
pub(super) fn new() -> Self {
let dir = tempfile::tempdir().unwrap();
git_in(dir.path(), &["init", "--quiet", "-b", "main"]);
Self { dir }
}
fn path(&self) -> PathBuf {
pub(super) fn path(&self) -> PathBuf {
self.dir.path().to_path_buf()
}
/// Writes files, removes paths, commits everything and returns the commit.
fn commit(&self, write: &[(&str, &[u8])], remove: &[&str]) -> String {
pub(super) fn commit(&self, write: &[(&str, &[u8])], remove: &[&str]) -> String {
for (path, bytes) in write {
let full = self.dir.path().join(path);
std::fs::create_dir_all(full.parent().unwrap()).unwrap();
@ -56,13 +56,13 @@ impl Remote {
}
}
struct FakeForge {
pub(super) struct FakeForge {
repos: Vec<(SourceRepo, PathBuf)>,
shortfall: Option<String>,
}
impl FakeForge {
fn of(repos: &[(&str, &Remote)]) -> Self {
pub(super) fn of(repos: &[(&str, &Remote)]) -> Self {
Self {
repos: repos
.iter()

View file

@ -120,10 +120,18 @@ k=60 are calibration, not design; RRF over a handful of lists is the
established baseline and I did not survey learned fusion (it needs training
data we lack).
## Status (head-maintainer pass, 2026-11-05, Huginn `25199c9`)
## Status (2026-11-17)
Checked against main on merge. Nothing in this map is built: no `texts` on
`SourceQuery`, no chat code under `sources/`. The anchors above were corrected.
Built: cut 1 (`texts` on `SourceQuery`), chat search and window, the chat ingest
core (`sources/chat.rs`, `room_scope.rs`, `chat_purge.rs`) and the ingest job.
The job is `huginn-sources`: with `--matrix-url`, `--matrix-token-file`, three
`--chat-exclude-room` ids (the estate rooms) and `--lock-file` it runs the forge
refresh and then `refresh_chat` through `sources/matrix.rs` (GET only, bearer
from a systemd credential file); without them it retires every chat point. The
rooms indexed are the reader's memberships; no flag lists rooms. `huginn-sources
chat-purge` writes a steward's purge tombstone under the same lock. What the
operator still has to do (reader account, credential, roster, unit lines) is the
handoff list of cut `gamecult-chat:cut_spec:cut-chat-ingest-job`.
What changed since the map was written:
- Questions 1 and 2 are answered with the recommendations