Chat ingest job: huginn-sources chat flags, Matrix reader, lock, chat-purge (#67)
This commit is contained in:
parent
64e88966ad
commit
07942c1a83
13 changed files with 1267 additions and 143 deletions
|
|
@ -18,7 +18,7 @@ name = "huginn-daemon"
|
|||
path = "src/main.rs"
|
||||
|
||||
# The source index's one writer: a oneshot job that brings `huginn_sources` to
|
||||
# the forge's current state and exits. No daemon calls any of its writes.
|
||||
# the forge's and the chat reader's current state and exits. No daemon calls any of its writes.
|
||||
[[bin]]
|
||||
name = "huginn-sources"
|
||||
path = "src/bin/huginn-sources.rs"
|
||||
|
|
@ -34,7 +34,8 @@ rmp-serde = "1"
|
|||
rustix = { version = "1", default-features = false, features = ["std", "net"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
# The vector store's and the embedder's HTTP APIs are the xenos boundary, and
|
||||
# JSON exists in `index/ollama.rs` and `index/qdrant.rs` only. `ureq` 3 with
|
||||
# JSON exists in `index/ollama.rs`, `index/qdrant.rs`, `sources/forge.rs` and
|
||||
# `sources/matrix.rs` only. `ureq` 3 with
|
||||
# default features off is plain HTTP with no TLS (`rustls` and `gzip` are its
|
||||
# only defaults), which is all the private mesh needs. It stays below 3.4,
|
||||
# whose `base64` 0.23 would put a second `base64` in a tree that has one.
|
||||
|
|
|
|||
|
|
@ -1,21 +1,38 @@
|
|||
//! The source index's refresh job: one run brings the `huginn_sources`
|
||||
//! collection to the forge's current state, journals one line per repo, and
|
||||
//! exits nonzero if any repo failed or the collection could not be used.
|
||||
//! The source index's refresh job: one run brings the forge corpus and, when
|
||||
//! configured, the rooms the chat reader has joined (minus the estate and DM
|
||||
//! exclusion) to current state, journals one line per repo and room, and exits
|
||||
//! nonzero if either failed or the collection could not be used. The
|
||||
//! `chat-purge` subcommand writes a steward's purge tombstone and nothing else.
|
||||
//!
|
||||
//! Exit status: 0 clean, 1 a run failed, 2 bad arguments, 3 another run holds
|
||||
//! the lock. The Matrix token is read from a file (a systemd credential),
|
||||
//! never from the environment or argv.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::PathBuf;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::{File, OpenOptions, TryLockError};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
|
||||
use anyhow::{Context, Result, ensure};
|
||||
use chrono::{DateTime, Utc};
|
||||
use huginn_daemon::index::ollama::pool;
|
||||
use huginn_daemon::index::qdrant::QdrantIndex;
|
||||
use huginn_daemon::sources::forge::HttpForge;
|
||||
use huginn_daemon::sources::refresh;
|
||||
use huginn_daemon::sources::job::{self, Chat, Purge};
|
||||
use huginn_daemon::sources::matrix::HttpMatrix;
|
||||
use huginn_daemon::sources::room_scope::{ESTATE_ALIASES, Estate};
|
||||
|
||||
/// Where the forge answers when the flag is absent: the loopback address of the
|
||||
/// forge on the host that runs this job.
|
||||
const DEFAULT_FORGE_URL: &str = "http://127.0.0.1:3310";
|
||||
|
||||
/// The chat flags, which come all together or not at all.
|
||||
struct ChatOptions {
|
||||
matrix_url: String,
|
||||
token: String,
|
||||
estate: Estate,
|
||||
}
|
||||
|
||||
struct Options {
|
||||
state_root: PathBuf,
|
||||
qdrant_url: String,
|
||||
|
|
@ -25,75 +42,310 @@ struct Options {
|
|||
reserve_urls: Vec<String>,
|
||||
embedding_model: String,
|
||||
forge_url: String,
|
||||
chat: Option<ChatOptions>,
|
||||
lock_file: Option<PathBuf>,
|
||||
}
|
||||
|
||||
fn parse(args: impl Iterator<Item = String>) -> Result<Options> {
|
||||
enum Command {
|
||||
Run(Options),
|
||||
Purge { qdrant_url: String, lock_file: PathBuf, purge: Purge },
|
||||
}
|
||||
|
||||
/// How a run ends other than by finishing.
|
||||
enum Stop {
|
||||
Usage(anyhow::Error),
|
||||
Locked,
|
||||
}
|
||||
|
||||
/// The options named, each once, except those in `repeated`, which keep every value.
|
||||
fn collect(args: impl Iterator<Item = String>, once: &[&str], repeated: &[&str]) -> Result<(BTreeMap<String, String>, BTreeMap<String, Vec<String>>)> {
|
||||
let mut args = args;
|
||||
let mut values: BTreeMap<String, String> = BTreeMap::new();
|
||||
let mut ollama_urls = Vec::new();
|
||||
let mut reserve_urls = Vec::new();
|
||||
let mut singles = BTreeMap::new();
|
||||
let mut many: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||
while let Some(name) = args.next() {
|
||||
let name = name.strip_prefix("--").with_context(|| format!("expected --option, got {name:?}"))?;
|
||||
ensure!(
|
||||
matches!(name, "state-root" | "qdrant-url" | "ollama-url" | "reserve-ollama-url" | "embedding-model" | "forge-url"),
|
||||
"unsupported huginn-sources option --{name}"
|
||||
);
|
||||
ensure!(once.contains(&name) || repeated.contains(&name), "unsupported huginn-sources option --{name}");
|
||||
let value = args.next().with_context(|| format!("missing value for --{name}"))?;
|
||||
if name == "ollama-url" || name == "reserve-ollama-url" {
|
||||
ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}");
|
||||
if name == "ollama-url" { ollama_urls.push(value) } else { reserve_urls.push(value) }
|
||||
if repeated.contains(&name) {
|
||||
many.entry(name.to_owned()).or_default().push(value);
|
||||
} else {
|
||||
ensure!(values.insert(name.to_owned(), value).is_none(), "duplicate option --{name}");
|
||||
ensure!(singles.insert(name.to_owned(), value).is_none(), "duplicate option --{name}");
|
||||
}
|
||||
}
|
||||
let take = |name: &str| values.get(name).cloned().with_context(|| format!("--{name} is required"));
|
||||
let url = |name: &str, value: String| {
|
||||
ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}");
|
||||
Ok(value)
|
||||
};
|
||||
let state_root = PathBuf::from(take("state-root")?);
|
||||
ensure!(state_root.is_absolute(), "--state-root must be an absolute path");
|
||||
let embedding_model = take("embedding-model")?;
|
||||
ensure!(!embedding_model.is_empty(), "--embedding-model must name a model");
|
||||
Ok(Options {
|
||||
state_root,
|
||||
qdrant_url: url("qdrant-url", take("qdrant-url")?)?,
|
||||
ollama_urls: {
|
||||
ensure!(!ollama_urls.is_empty(), "at least one --ollama-url is required; repeat it for each GPU host");
|
||||
ollama_urls
|
||||
Ok((singles, many))
|
||||
}
|
||||
|
||||
fn http(name: &str, value: String) -> Result<String> {
|
||||
ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}");
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// The Matrix reader's bearer travels in clear, and this build speaks no TLS: only a loopback host may be named.
|
||||
fn loopback_http(name: &str, value: String) -> Result<String> {
|
||||
let value = http(name, value)?;
|
||||
let authority = value["http://".len()..].split(['/', '?', '#']).next().unwrap_or("");
|
||||
let host = match authority.strip_prefix('[') {
|
||||
// The bracket must end the authority or be followed by the port, so `[::1].evil.example` names no loopback.
|
||||
Some(inner) => match inner.split_once(']') {
|
||||
Some((host, "")) => host,
|
||||
Some((host, port)) if port.starts_with(':') => host,
|
||||
_ => "",
|
||||
},
|
||||
None => authority.split(':').next().unwrap_or(""),
|
||||
};
|
||||
let loopback = !authority.contains('@') && (host.eq_ignore_ascii_case("localhost") || host.parse::<std::net::IpAddr>().is_ok_and(|ip| ip.is_loopback()));
|
||||
ensure!(loopback, "--{name} must name a loopback host (localhost, 127.0.0.0/8 or [::1]): the bearer token travels in clear and this build has no TLS, got {value:?}");
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
fn absolute(name: &str, value: &str) -> Result<PathBuf> {
|
||||
let path = PathBuf::from(value);
|
||||
ensure!(path.is_absolute(), "--{name} must be an absolute path");
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn chat_options(singles: &BTreeMap<String, String>, excluded: Vec<String>) -> Result<Option<ChatOptions>> {
|
||||
let given = [singles.contains_key("matrix-url"), singles.contains_key("matrix-token-file"), !excluded.is_empty()];
|
||||
if !given.contains(&true) {
|
||||
return Ok(None);
|
||||
}
|
||||
for (present, name) in given.iter().zip(["matrix-url", "matrix-token-file", "chat-exclude-room"]) {
|
||||
ensure!(*present, "--{name} is missing: the chat flags (--matrix-url, --matrix-token-file, --chat-exclude-room) come all together or not at all");
|
||||
}
|
||||
let ids: BTreeSet<String> = excluded.iter().cloned().collect();
|
||||
ensure!(ids.len() == excluded.len(), "--chat-exclude-room must not repeat a room id");
|
||||
ensure!(ids.len() == ESTATE_ALIASES.len(), "--chat-exclude-room must be given exactly {} times (the ids of {})", ESTATE_ALIASES.len(), ESTATE_ALIASES.join(", "));
|
||||
ensure!(ids.iter().all(|id| id.starts_with('!')), "--chat-exclude-room takes room ids beginning with !, not aliases");
|
||||
let path = absolute("matrix-token-file", &singles["matrix-token-file"])?;
|
||||
let token = std::fs::read_to_string(&path).with_context(|| format!("reading --matrix-token-file {}", path.display()))?.trim().to_owned();
|
||||
ensure!(!token.is_empty(), "--matrix-token-file {} is empty", path.display());
|
||||
Ok(Some(ChatOptions { matrix_url: loopback_http("matrix-url", singles["matrix-url"].clone())?, token, estate: Estate::new(ids)? }))
|
||||
}
|
||||
|
||||
fn parse(mut args: impl Iterator<Item = String>, now: DateTime<Utc>) -> Result<Command> {
|
||||
let mut rest: Vec<String> = Vec::new();
|
||||
let first = args.next();
|
||||
if first.as_deref() == Some("chat-purge") {
|
||||
let (singles, _) = collect(args, &["qdrant-url", "room", "from", "to", "request", "lock-file"], &[])?;
|
||||
let take = |name: &str| singles.get(name).cloned().with_context(|| format!("--{name} is required"));
|
||||
return Ok(Command::Purge {
|
||||
qdrant_url: http("qdrant-url", take("qdrant-url")?)?,
|
||||
lock_file: absolute("lock-file", &take("lock-file")?)?,
|
||||
purge: Purge::check(&take("room")?, &take("from")?, &take("to")?, &take("request")?, now)?,
|
||||
});
|
||||
}
|
||||
rest.extend(first);
|
||||
let (mut singles, mut many) = collect(
|
||||
rest.into_iter().chain(args),
|
||||
&["state-root", "qdrant-url", "embedding-model", "forge-url", "matrix-url", "matrix-token-file", "lock-file"],
|
||||
&["ollama-url", "reserve-ollama-url", "chat-exclude-room"],
|
||||
)?;
|
||||
let mut hosts = |name: &str| -> Result<Vec<String>> { many.remove(name).unwrap_or_default().into_iter().map(|url| http(name, url)).collect() };
|
||||
let ollama_urls = hosts("ollama-url")?;
|
||||
let reserve_urls = hosts("reserve-ollama-url")?;
|
||||
let excluded = many.remove("chat-exclude-room").unwrap_or_default();
|
||||
let take = |singles: &mut BTreeMap<String, String>, name: &str| singles.remove(name).with_context(|| format!("--{name} is required"));
|
||||
let state_root = absolute("state-root", &take(&mut singles, "state-root")?)?;
|
||||
let embedding_model = take(&mut singles, "embedding-model")?;
|
||||
ensure!(!embedding_model.is_empty(), "--embedding-model must name a model");
|
||||
ensure!(!ollama_urls.is_empty(), "at least one --ollama-url is required; repeat it for each GPU host");
|
||||
let chat = chat_options(&singles, excluded)?;
|
||||
let lock_file = singles.get("lock-file").map(|path| absolute("lock-file", path)).transpose()?;
|
||||
ensure!(chat.is_none() || lock_file.is_some(), "--lock-file is required whenever the chat flags are given");
|
||||
Ok(Command::Run(Options {
|
||||
state_root,
|
||||
qdrant_url: http("qdrant-url", take(&mut singles, "qdrant-url")?)?,
|
||||
ollama_urls,
|
||||
reserve_urls,
|
||||
embedding_model,
|
||||
forge_url: url("forge-url", values.get("forge-url").cloned().unwrap_or_else(|| DEFAULT_FORGE_URL.to_owned()))?,
|
||||
})
|
||||
forge_url: http("forge-url", singles.remove("forge-url").unwrap_or_else(|| DEFAULT_FORGE_URL.to_owned()))?,
|
||||
chat,
|
||||
lock_file,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Takes the job's lock for the life of the returned file. A held lock means a run (or a
|
||||
/// purge request) is in progress and nothing may change: the cursors object is replaced whole.
|
||||
fn take_lock(path: &Path) -> Result<File, Stop> {
|
||||
let file = OpenOptions::new().create(true).truncate(false).write(true).open(path).with_context(|| format!("opening --lock-file {}", path.display())).map_err(Stop::Usage)?;
|
||||
match file.try_lock() {
|
||||
Ok(()) => Ok(file),
|
||||
Err(TryLockError::WouldBlock) => Err(Stop::Locked),
|
||||
Err(TryLockError::Error(error)) => Err(Stop::Usage(anyhow::Error::new(error).context(format!("locking --lock-file {}", path.display())))),
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_if(path: &Option<PathBuf>) -> Result<Option<File>, Stop> {
|
||||
path.as_deref().map(take_lock).transpose()
|
||||
}
|
||||
|
||||
fn execute(command: Command) -> Result<u8, Stop> {
|
||||
match command {
|
||||
Command::Purge { qdrant_url, lock_file, purge } => {
|
||||
let _lock = take_lock(&lock_file)?;
|
||||
let mut index = QdrantIndex::new(&qdrant_url);
|
||||
Ok(match job::purge(&mut index, &purge) {
|
||||
Ok(line) => {
|
||||
println!("{line}");
|
||||
0
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!("huginn-sources: {error:#}");
|
||||
1
|
||||
}
|
||||
})
|
||||
}
|
||||
Command::Run(options) => {
|
||||
let _lock = lock_if(&options.lock_file)?;
|
||||
let mut embedder = pool(&options.ollama_urls, &options.reserve_urls, &options.embedding_model);
|
||||
let mut index = QdrantIndex::new(&options.qdrant_url);
|
||||
let mut forge = HttpForge::new(&options.forge_url);
|
||||
let mut matrix = options.chat.as_ref().map(|chat| HttpMatrix::new(&chat.matrix_url, &chat.token));
|
||||
let chat = matrix.as_mut().zip(options.chat.as_ref()).map(|(history, chat)| Chat { history, estate: &chat.estate });
|
||||
Ok(job::run(&mut embedder, &mut index, &mut forge, &options.state_root, chat, &mut |line| println!("{line}"), &mut |line| eprintln!("{line}")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let options = match parse(std::env::args().skip(1)) {
|
||||
Ok(options) => options,
|
||||
Err(error) => {
|
||||
match parse(std::env::args().skip(1), Utc::now()).map_err(Stop::Usage).and_then(execute) {
|
||||
Ok(status) => ExitCode::from(status),
|
||||
Err(Stop::Usage(error)) => {
|
||||
eprintln!("huginn-sources: {error:#}");
|
||||
return ExitCode::from(2);
|
||||
ExitCode::from(2)
|
||||
}
|
||||
};
|
||||
let mut embedder = pool(&options.ollama_urls, &options.reserve_urls, &options.embedding_model);
|
||||
let mut index = QdrantIndex::new(&options.qdrant_url);
|
||||
let mut forge = HttpForge::new(&options.forge_url);
|
||||
let mut journal = |report: &huginn_daemon::sources::Report| println!("{report}");
|
||||
match refresh(&mut embedder, &mut index, &mut forge, &options.state_root, &mut journal) {
|
||||
Ok(refreshed) if refreshed.is_clean() => ExitCode::SUCCESS,
|
||||
Ok(refreshed) => {
|
||||
if !refreshed.failed.is_empty() {
|
||||
eprintln!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", "));
|
||||
}
|
||||
if let Some(why) = &refreshed.retirement_withheld {
|
||||
eprintln!("huginn-sources: no repo was retired: {why}");
|
||||
}
|
||||
ExitCode::from(1)
|
||||
}
|
||||
Err(error) => {
|
||||
eprintln!("huginn-sources: {error:#}");
|
||||
ExitCode::from(1)
|
||||
Err(Stop::Locked) => {
|
||||
eprintln!("huginn-sources: huginn-sources is running");
|
||||
ExitCode::from(3)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const NOW: &str = "2026-11-17T00:00:00Z";
|
||||
|
||||
fn args(text: &str) -> impl Iterator<Item = String> {
|
||||
text.split_whitespace().map(str::to_owned).collect::<Vec<_>>().into_iter()
|
||||
}
|
||||
|
||||
fn now() -> DateTime<Utc> {
|
||||
DateTime::parse_from_rfc3339(NOW).unwrap().with_timezone(&Utc)
|
||||
}
|
||||
|
||||
fn error(text: &str) -> String {
|
||||
match parse(args(text), now()) {
|
||||
Ok(_) => panic!("parsed: {text}"),
|
||||
Err(error) => format!("{error:#}"),
|
||||
}
|
||||
}
|
||||
|
||||
const BASE: &str = "--state-root /s --qdrant-url http://q --ollama-url http://o --embedding-model m";
|
||||
const IDS: &str = "--chat-exclude-room !a:s --chat-exclude-room !b:s --chat-exclude-room !c:s";
|
||||
|
||||
fn with_token(secret: &str) -> (tempfile::TempDir, String) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("token");
|
||||
std::fs::write(&path, secret).unwrap();
|
||||
let path = path.display().to_string();
|
||||
(dir, path)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_flags_come_all_or_none() {
|
||||
let (_dir, token) = with_token(" sekret-token\n");
|
||||
let lock = "--lock-file /l";
|
||||
let full = format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS} {lock}");
|
||||
let Ok(Command::Run(options)) = parse(args(&full), now()) else { panic!("full chat flags must parse") };
|
||||
assert_eq!(options.chat.as_ref().unwrap().token, "sekret-token", "trimmed");
|
||||
assert!(matches!(parse(args(BASE), now()), Ok(Command::Run(Options { chat: None, lock_file: None, .. }))));
|
||||
assert!(error(&format!("{BASE} --matrix-url http://localhost {lock}")).contains("--matrix-token-file is missing"));
|
||||
assert!(error(&format!("{BASE} --matrix-token-file {token} {IDS} {lock}")).contains("--matrix-url is missing"));
|
||||
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {lock}")).contains("--chat-exclude-room is missing"));
|
||||
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS}")).contains("--lock-file is required"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_flag_values_are_checked_and_never_print_the_token() {
|
||||
let (_dir, token) = with_token("sekret-token");
|
||||
let on = |extra: &str| format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} --lock-file /l {extra}");
|
||||
assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !b:s")).contains("exactly 3"));
|
||||
assert!(error(&on(&format!("{IDS} --chat-exclude-room !d:s"))).contains("exactly 3"));
|
||||
assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !a:s --chat-exclude-room !b:s")).contains("repeat"));
|
||||
assert!(error(&on("--chat-exclude-room !a:s --chat-exclude-room !b:s --chat-exclude-room #c:s")).contains("beginning with !"));
|
||||
assert!(error(&format!("{BASE} --matrix-url https://m --matrix-token-file {token} --lock-file /l {IDS}")).contains("plain http://"));
|
||||
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file rel --lock-file /l {IDS}")).contains("absolute"));
|
||||
assert!(error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} --lock-file rel {IDS}")).contains("absolute"));
|
||||
let (_empty, empty) = with_token(" \n");
|
||||
let message = error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {empty} --lock-file /l {IDS}"));
|
||||
assert!(message.contains(&empty) && message.contains("empty"));
|
||||
let missing = error(&format!("{BASE} --matrix-url http://localhost --matrix-token-file /nowhere/t --lock-file /l {IDS}"));
|
||||
assert!(missing.contains("/nowhere/t"));
|
||||
for text in [on(IDS), on("--chat-exclude-room !a:s")] {
|
||||
assert!(!error(&format!("{text} --nope x")).contains("sekret-token"));
|
||||
}
|
||||
let dead = ["--chat", "room"].join("-");
|
||||
assert!(error(&format!("{BASE} {dead} !x:s")).contains(&format!("unsupported huginn-sources option {dead}")));
|
||||
}
|
||||
|
||||
fn purge(extra: &str) -> String {
|
||||
error(&format!("chat-purge --qdrant-url http://q --lock-file /l {extra}"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_purge_arguments() {
|
||||
let good = "--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z --request ref-1";
|
||||
let Ok(Command::Purge { purge: asked, .. }) = parse(args(&format!("chat-purge --qdrant-url http://q --lock-file /l {good}")), now()) else { panic!() };
|
||||
assert_eq!((asked.window.from_ms, asked.window.to_ms), (1_794_268_800_000, 1_794_355_200_000));
|
||||
let bad = |from: &str, to: &str, room: &str, request: &str| purge(&format!("--room {room} --from {from} --to {to} --request {request}"));
|
||||
let (f, t) = ("2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z");
|
||||
assert!(bad(f, t, "#a:s", "r").contains("--room"));
|
||||
assert!(bad("2026-11-10T00:00:00+00:00", t, "!r:s", "r").contains("--from"));
|
||||
assert!(bad(f, "2026-11-11T00:00:00", "!r:s", "r").contains("--to"));
|
||||
assert!(bad(t, f, "!r:s", "r").contains("before"));
|
||||
assert!(bad(f, "2026-11-18T00:00:00Z", "!r:s", "r").contains("after now"));
|
||||
assert!(bad("2025-11-09T00:00:00Z", t, "!r:s", "r").contains("366"));
|
||||
assert!(purge(&format!("--room !r:s --from {f} --to {t} --request {}", "x".repeat(121))).contains("--request"));
|
||||
assert!(purge("--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z").contains("--request is required"));
|
||||
assert!(parse(args(&format!("chat-purge --qdrant-url http://q --lock-file /l {good} --matrix-url http://localhost")), now()).is_err(), "no Matrix flags");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matrix_url_must_be_loopback_http() {
|
||||
let (_dir, token) = with_token("sekret-token");
|
||||
let with = |url: &str| format!("{BASE} --matrix-url {url} --matrix-token-file {token} --lock-file /l {IDS}");
|
||||
for good in ["http://localhost", "http://127.0.0.1:8008", "http://127.9.9.9/", "http://[::1]:8008", "http://LOCALHOST:1/x"] {
|
||||
assert!(parse(args(&with(good)), now()).is_ok(), "{good}");
|
||||
}
|
||||
for bad in ["http://matrix.example", "http://10.0.0.5:8008", "http://localhost@evil.example", "http://127.0.0.1.evil.example", "http://localhost.evil.example", "http://[::2]", "https://localhost", "http://127.0.0.1:pw@evil.example", "http://[::1].evil.example", "http://[::1]x"] {
|
||||
assert!(error(&with(bad)).contains("--matrix-url"), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_lock_exits_3_through_execute_for_both_modes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let lock = dir.path().join("lock");
|
||||
let held = take_lock(&lock).ok().unwrap();
|
||||
let lock = lock.display().to_string();
|
||||
let good = "--room !r:s --from 2026-11-10T00:00:00Z --to 2026-11-11T00:00:00Z --request r";
|
||||
let purge = parse(args(&format!("chat-purge --qdrant-url http://127.0.0.1:1 --lock-file {lock} {good}")), now()).unwrap();
|
||||
assert!(matches!(execute(purge), Err(Stop::Locked)));
|
||||
let (_t, token) = with_token("sekret-token");
|
||||
let run = parse(args(&format!("{BASE} --matrix-url http://localhost --matrix-token-file {token} {IDS} --lock-file {lock}")), now()).unwrap();
|
||||
assert!(matches!(execute(run), Err(Stop::Locked)));
|
||||
drop(held);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_lock_stops_the_run_untouched() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("lock");
|
||||
let held = take_lock(&path).ok().unwrap();
|
||||
assert!(matches!(take_lock(&path), Err(Stop::Locked)));
|
||||
drop(held);
|
||||
assert!(take_lock(&path).is_ok());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -134,6 +134,39 @@ impl std::fmt::Display for ChatReport {
|
|||
}
|
||||
}
|
||||
|
||||
/// One line of a chat run's journal, typed so the job routes by what happened and not by how the line reads.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum Entry {
|
||||
/// A room's report; a failed one is an alarm.
|
||||
Report(ChatReport),
|
||||
Refused(RefusedRoom),
|
||||
Scrubbed(ScrubHit),
|
||||
/// A failure outside any one room's report: retiring rooms.
|
||||
Trouble(String),
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
/// The operator must see this on stderr: a failure, a refusal, a scrub hit.
|
||||
pub fn is_alarm(&self) -> bool {
|
||||
match self {
|
||||
Self::Report(report) => report.failure.is_some(),
|
||||
Self::Refused(_) | Self::Scrubbed(_) | Self::Trouble(_) => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Entry {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Report(report) => report.fmt(formatter),
|
||||
Self::Refused(RefusedRoom { room_id, reason, event_id: None }) => write!(formatter, "refused: {room_id} {reason}"),
|
||||
Self::Refused(RefusedRoom { room_id, reason, event_id: Some(event) }) => write!(formatter, "refused: {room_id} {reason} [{event}]"),
|
||||
Self::Scrubbed(hit) => write!(formatter, "scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind),
|
||||
Self::Trouble(why) => write!(formatter, "huginn-sources: {why}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A joined room the scope refused, whole, and why; `event_id` when an event did it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RefusedRoom {
|
||||
|
|
@ -562,7 +595,7 @@ pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
|||
index: &mut S,
|
||||
history: &mut H,
|
||||
estate: &Estate,
|
||||
journal: &mut dyn FnMut(String),
|
||||
journal: &mut dyn FnMut(Entry),
|
||||
) -> Result<ChatRefreshed> {
|
||||
let DescribedSources::Labelled { meta, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||
bail!("the source collection is absent or not Huginn's; the forge refresh creates it and chat never does");
|
||||
|
|
@ -589,13 +622,14 @@ pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
|||
Ok(Err(reason)) => {
|
||||
// Not kept, so the retirement below purges whatever it held.
|
||||
keep.remove(room_id);
|
||||
journal(format!("refused: {room_id} {reason}"));
|
||||
out.refused.push(RefusedRoom { room_id: room_id.clone(), reason, event_id: None });
|
||||
let refused = RefusedRoom { room_id: room_id.clone(), reason, event_id: None };
|
||||
journal(Entry::Refused(refused.clone()));
|
||||
out.refused.push(refused);
|
||||
}
|
||||
// An error is not a reclassification: the room keeps its points and cursor, and the report says why.
|
||||
Err(error) => {
|
||||
let report = ChatReport::failed(room_id, None, &error.context("reading the room's facts"));
|
||||
journal(report.to_string());
|
||||
journal(Entry::Report(report.clone()));
|
||||
out.failed.push(room_id.clone());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
|
|
@ -604,35 +638,35 @@ pub fn refresh_chat<E: Embedder, S: SourceIndex, H: RoomHistory>(
|
|||
match retire_chat(index, &keep) {
|
||||
Ok(done) => {
|
||||
for (room, why) in done.failed {
|
||||
journal(format!("huginn-sources: retiring {room} failed, the others go on: {why}"));
|
||||
journal(Entry::Trouble(format!("retiring {room} failed, the others go on: {why}")));
|
||||
out.failed.push(room);
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
journal(format!("huginn-sources: retiring rooms failed: {error:#}"));
|
||||
journal(Entry::Trouble(format!("retiring rooms failed: {error:#}")));
|
||||
out.failed.push("retire".to_owned());
|
||||
}
|
||||
}
|
||||
for facts in admitted {
|
||||
match ingest_room(embedder, index, history, identity.dimensions, &facts, &mut out) {
|
||||
Ok(Outcome::Done(report)) => {
|
||||
journal(report.to_string());
|
||||
journal(Entry::Report(report.clone()));
|
||||
out.rooms.push(report);
|
||||
}
|
||||
Ok(Outcome::Refused(refused)) => {
|
||||
journal(format!("refused: {} {} [{}]", refused.room_id, refused.reason, refused.event_id.as_deref().unwrap_or("")));
|
||||
journal(Entry::Refused(refused.clone()));
|
||||
out.refused.push(refused);
|
||||
}
|
||||
Err(error) => {
|
||||
let report = ChatReport::failed(&facts.room_id, facts.alias.clone(), &error);
|
||||
journal(report.to_string());
|
||||
journal(Entry::Report(report.clone()));
|
||||
out.failed.push(facts.room_id.clone());
|
||||
out.rooms.push(report);
|
||||
}
|
||||
}
|
||||
}
|
||||
for hit in &out.scrubbed {
|
||||
journal(format!("scrubbed: {} {} {}", hit.room_id, hit.event_id, hit.kind));
|
||||
journal(Entry::Scrubbed(hit.clone()));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ impl Rig {
|
|||
|
||||
fn run(&mut self, history: &mut FakeHistory) -> (Result<ChatRefreshed>, Vec<String>) {
|
||||
let mut journal = Vec::new();
|
||||
let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line));
|
||||
let out = refresh_chat(&mut self.embedder, &mut self.index, history, &self.estate, &mut |line| journal.push(line.to_string()));
|
||||
(out, journal)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -322,6 +322,7 @@ mod tests {
|
|||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
use crate::sources::test_http::{Reply, forge_of};
|
||||
|
||||
/// Only public, live, original, non-empty repos are the corpus; a repo the
|
||||
/// forge does not say is safe is not indexed.
|
||||
|
|
@ -400,60 +401,6 @@ mod tests {
|
|||
assert!(HttpForge::new(&unstated).catalog().unwrap().shortfall.unwrap().contains("X-Total-Count"));
|
||||
}
|
||||
|
||||
/// What a fake forge answers: a status, the `Location` header when there is one, and a body.
|
||||
struct Reply {
|
||||
status: u16,
|
||||
location: Option<String>,
|
||||
body: String,
|
||||
}
|
||||
|
||||
impl From<String> for Reply {
|
||||
fn from(body: String) -> Self {
|
||||
Self { status: 200, location: None, body }
|
||||
}
|
||||
}
|
||||
|
||||
impl Reply {
|
||||
/// What the live forge answers an anonymous reader for a private repo.
|
||||
fn not_found() -> Self {
|
||||
Self { status: 404, location: None, body: json!({ "message": "The target couldn't be found." }).to_string() }
|
||||
}
|
||||
|
||||
/// What the live forge answers for a renamed repo: a 301 to its public address, `/forge` prefix and all.
|
||||
fn moved_to(location: &str) -> Self {
|
||||
Self { status: 301, location: Some(location.to_owned()), body: String::new() }
|
||||
}
|
||||
}
|
||||
|
||||
/// Answers each request by its path from `routes`; a path it has no answer for is held open and never answered, as a hung forge
|
||||
/// does. Every path asked is recorded in order.
|
||||
fn forge_of<R: Into<Reply>>(routes: impl Fn(&str) -> Option<R> + Send + 'static) -> (String, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
|
||||
use std::io::{Read, Write};
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let asked = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let record = std::sync::Arc::clone(&asked);
|
||||
std::thread::spawn(move || {
|
||||
let mut held = Vec::new();
|
||||
for stream in listener.incoming() {
|
||||
let mut stream = stream.unwrap();
|
||||
let mut buffer = [0_u8; 4096];
|
||||
let read = stream.read(&mut buffer).unwrap();
|
||||
let request = String::from_utf8_lossy(&buffer[..read]).into_owned();
|
||||
let path = request.split_whitespace().nth(1).unwrap_or_default().to_owned();
|
||||
record.lock().unwrap().push(path.clone());
|
||||
match routes(&path).map(Into::into) {
|
||||
Some(Reply { status, location, body }) => {
|
||||
let location = location.map(|location| format!("location: {location}\r\n")).unwrap_or_default();
|
||||
write!(stream, "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\n{location}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap();
|
||||
}
|
||||
None => held.push(stream),
|
||||
}
|
||||
}
|
||||
});
|
||||
(base, asked)
|
||||
}
|
||||
|
||||
const FEED_SAMPLE: &str = include_str!("../../tests/fixtures/feed.json");
|
||||
|
||||
const NOW: &str = "2026-11-03T12:00:00Z";
|
||||
|
|
@ -523,7 +470,7 @@ mod tests {
|
|||
assert!(read.open_prs.is_empty() && read.unseen.is_empty() && read.capped.is_empty() && read.moved.is_empty(), "a cursor asks for neither list");
|
||||
let wire = serde_json::to_string(&read).unwrap();
|
||||
assert!(!wire.contains("CANARY") && !wire.contains("obey") && !wire.contains("Message"), "no forge text leaves the read: {wire}");
|
||||
let asked = asked.lock().unwrap().clone();
|
||||
let asked = asked.paths();
|
||||
assert!(!asked.is_empty() && asked.iter().all(|path| names.iter().any(|name| path.starts_with(&format!("/api/v1/repos/{name}/activities/feeds")))), "{asked:?}");
|
||||
}
|
||||
|
||||
|
|
@ -555,13 +502,13 @@ mod tests {
|
|||
let far = HttpForge::new(&base).activity(&wanted, &me, Some(5), now).unwrap();
|
||||
assert!(far.capped == repos(&["GameCult/Huginn"]), "stopped at the cap with the cursor not reached");
|
||||
assert_eq!((far.events.len(), far.head), (200, 2500));
|
||||
assert_eq!(asked.lock().unwrap().len(), 4, "four pages and no more");
|
||||
assert_eq!(asked.paths().len(), 4, "four pages and no more");
|
||||
|
||||
let (base, asked) = forge_of(feed("nibu"));
|
||||
let near = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap();
|
||||
assert!(near.capped.is_empty());
|
||||
assert_eq!((near.events.len(), near.head), (20, 2500), "ids above the cursor only");
|
||||
assert_eq!(asked.lock().unwrap().len(), 1);
|
||||
assert_eq!(asked.paths().len(), 1);
|
||||
|
||||
let (base, _) = forge_of(feed("eureka"));
|
||||
let hers = HttpForge::new(&base).activity(&wanted, &me, Some(2300), now).unwrap();
|
||||
|
|
@ -583,7 +530,7 @@ mod tests {
|
|||
let read = HttpForge::new(&base).activity(&repos(&["GameCult/Ratatoskr"]), &me(), None, at(NOW)).unwrap();
|
||||
assert_eq!(read.events.iter().map(|event| event.id).collect::<Vec<_>>(), [300]);
|
||||
assert!(read.capped.is_empty());
|
||||
assert!(asked.lock().unwrap().iter().all(|path| !path.contains("/orgs/GameCult/activities")));
|
||||
assert!(asked.paths().iter().all(|path| !path.contains("/orgs/GameCult/activities")));
|
||||
}
|
||||
|
||||
/// A first read keeps what is inside 72 hours and nothing older, and its cursor is the newest id it saw, an entry past the window
|
||||
|
|
@ -619,13 +566,13 @@ mod tests {
|
|||
let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) });
|
||||
let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap();
|
||||
assert_eq!((read.events.len(), read.capped.len()), (53, 0));
|
||||
assert_eq!(asked.lock().unwrap().iter().filter(|path| is_feed(path)).count(), 2, "page 2 was asked, page 3 was not");
|
||||
assert_eq!(asked.paths().iter().filter(|path| is_feed(path)).count(), 2, "page 2 was asked, page 3 was not");
|
||||
|
||||
let entries: Vec<Value> = (0..400).map(|n| push_at(5000 - n, "GameCult/Huginn", "Nibu", 5)).collect();
|
||||
let (base, asked) = forge_of(move |path| if is_feed(path) { repo_feed(&entries, path) } else { Some("[]".into()) });
|
||||
let read = HttpForge::new(&base).activity(&wanted, &me(), None, at(NOW)).unwrap();
|
||||
assert_eq!((read.events.len(), read.capped), (REPO_FEED_PAGES * PAGE, repos(&["GameCult/Huginn"])), "200 read and the rest disclosed");
|
||||
assert_eq!(asked.lock().unwrap().iter().filter(|path| is_feed(path)).count(), REPO_FEED_PAGES);
|
||||
assert_eq!(asked.paths().iter().filter(|path| is_feed(path)).count(), REPO_FEED_PAGES);
|
||||
}
|
||||
|
||||
/// With no cursor the read also lists the open PRs in her repos and hers anywhere, and the repos the forge's listing does not hold.
|
||||
|
|
@ -660,14 +607,14 @@ mod tests {
|
|||
assert_eq!(first.unseen, repos(&["GameCult/Ghost"]), "r60 is on page 2 of the listing");
|
||||
let window = at("2026-10-31T12:00:00Z").timestamp_millis() as u64;
|
||||
assert!(!first.events.is_empty() && first.events.iter().all(|event| event.at >= window), "inside 72 h: {:?}", first.events);
|
||||
let paths = asked.lock().unwrap().clone();
|
||||
let paths = asked.paths();
|
||||
let count = |what: &str| paths.iter().filter(|path| path.contains(what)).count();
|
||||
assert_eq!((count("/issues/search"), count("/orgs/GameCult/repos")), (2, 2), "a short page ends each: {paths:?}");
|
||||
|
||||
// A cursor asks for neither list.
|
||||
let before = paths.len();
|
||||
HttpForge::new(&base).activity(&wanted, &me(), Some(1), at(NOW)).unwrap();
|
||||
let after = asked.lock().unwrap().clone();
|
||||
let after = asked.paths();
|
||||
assert!(after[before..].iter().all(|path| is_feed(path)), "only the feeds: {after:?}");
|
||||
}
|
||||
|
||||
|
|
@ -693,12 +640,12 @@ mod tests {
|
|||
});
|
||||
let read = HttpForge::new(&base).activity(&repos(&["GameCult/Huginn"]), &me(), None, at(NOW)).unwrap();
|
||||
assert_eq!(read.open_prs.iter().map(|pr| pr.number).collect::<Vec<_>>(), [7], "page 3 was read");
|
||||
assert_eq!(asked.lock().unwrap().iter().filter(|path| path.contains("/issues/search")).count(), 4, "to the short page");
|
||||
assert_eq!(asked.paths().iter().filter(|path| path.contains("/issues/search")).count(), 4, "to the short page");
|
||||
}
|
||||
|
||||
/// The forge as it answers an anonymous reader: public repos give their feed (`[]` when quiet), a private repo answers 404, a
|
||||
/// renamed repo answers 301 to a path under `/forge` that this loopback does not serve (asking it gives 404).
|
||||
fn live_forge(entries: Vec<Value>, public: &'static [&'static str], listing: String) -> (String, std::sync::Arc<std::sync::Mutex<Vec<String>>>) {
|
||||
fn live_forge(entries: Vec<Value>, public: &'static [&'static str], listing: String) -> (String, crate::sources::test_http::Asked) {
|
||||
forge_of(move |path| {
|
||||
if path.starts_with("/forge/") {
|
||||
return Some(Reply::not_found());
|
||||
|
|
@ -733,7 +680,7 @@ mod tests {
|
|||
assert_eq!(first.unseen, repos(&["GameCult/gamecult-ops", "GameCult/cinny"]));
|
||||
assert_eq!(first.moved, [RepoMoved { from: "GameCult/cinny".into(), to: "GameCult/Ratatoskr".into() }]);
|
||||
assert!(first.capped.is_empty());
|
||||
assert!(asked.lock().unwrap().iter().all(|path| !path.starts_with("/forge/")), "a redirect is not followed: {:?}", asked.lock().unwrap());
|
||||
assert!(asked.paths().iter().all(|path| !path.starts_with("/forge/")), "a redirect is not followed: {:?}", asked.paths());
|
||||
|
||||
let later = forge.activity(&wanted, &me(), Some(250), at(NOW)).unwrap();
|
||||
assert_eq!((later.events.iter().map(|event| event.id).collect::<Vec<_>>(), later.head), (vec![300], 300), "a later read goes on past the dark repos");
|
||||
|
|
|
|||
128
crates/huginn-daemon/src/sources/job.rs
Normal file
128
crates/huginn-daemon/src/sources/job.rs
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
//! What one `huginn-sources` run does with the pieces it is handed: the forge
|
||||
//! refresh, then the chat refresh (or, with chat off, the retirement of every
|
||||
//! chat point), one exit status for both, and the steward-purge request. The
|
||||
//! binary supplies the real forge, embedder, vector store and Matrix client and
|
||||
//! owns the flags and the lock; this file decides nothing about scope or
|
||||
//! scrubbing (`room_scope` and `redact` do) and never sees a credential.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
use super::chat::{Entry, RoomHistory, refresh_chat, retire_chat};
|
||||
use super::chat_purge::{PurgeWindow, parse_windows, purge_key, request_purge};
|
||||
use super::room_scope::Estate;
|
||||
use super::{DescribedSources, Forge, SourceIndex, refresh};
|
||||
use crate::index::{Embedder, INDEX_CALL_TIMEOUT};
|
||||
|
||||
/// Chat on: the reader's rooms and the estate ids the scope refuses.
|
||||
pub struct Chat<'a, H> {
|
||||
pub history: &'a mut H,
|
||||
pub estate: &'a Estate,
|
||||
}
|
||||
|
||||
/// One run: the forge refresh, then chat (`Some`) or the retirement of every chat point (`None`).
|
||||
/// `say` takes the journal's report lines (stdout), `complain` the failures, refusals and scrub hits (stderr).
|
||||
/// A forge failure never skips the chat run and a chat failure never hides a forge one.
|
||||
/// Returns the exit status: 0 only when both were clean.
|
||||
pub fn run<E: Embedder, S: SourceIndex, F: Forge, H: RoomHistory>(
|
||||
embedder: &mut E,
|
||||
index: &mut S,
|
||||
forge: &mut F,
|
||||
state_root: &Path,
|
||||
chat: Option<Chat<'_, H>>,
|
||||
say: &mut dyn FnMut(String),
|
||||
complain: &mut dyn FnMut(String),
|
||||
) -> u8 {
|
||||
let mut clean = true;
|
||||
match refresh(embedder, index, forge, state_root, &mut |report| say(report.to_string())) {
|
||||
Ok(refreshed) if refreshed.is_clean() => {}
|
||||
Ok(refreshed) => {
|
||||
clean = false;
|
||||
if !refreshed.failed.is_empty() {
|
||||
complain(format!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", ")));
|
||||
}
|
||||
if let Some(why) = &refreshed.retirement_withheld {
|
||||
complain(format!("huginn-sources: no repo was retired: {why}"));
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
clean = false;
|
||||
complain(format!("huginn-sources: {error:#}"));
|
||||
}
|
||||
}
|
||||
match chat {
|
||||
Some(Chat { history, estate }) => {
|
||||
let mut line = |entry: Entry| if entry.is_alarm() { complain(entry.to_string()) } else { say(entry.to_string()) };
|
||||
match refresh_chat(embedder, index, history, estate, &mut line) {
|
||||
Ok(refreshed) => clean &= refreshed.is_clean(),
|
||||
Err(error) => {
|
||||
clean = false;
|
||||
complain(format!("huginn-sources: chat: {error:#}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
None => match retire_chat(index, &BTreeSet::new()) {
|
||||
Ok(done) => {
|
||||
if !done.retired.is_empty() {
|
||||
say(format!("chat retired: {} rooms", done.retired.len()));
|
||||
}
|
||||
for (room, why) in done.failed {
|
||||
clean = false;
|
||||
complain(format!("huginn-sources: retiring {room} failed: {why}"));
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
clean = false;
|
||||
complain(format!("huginn-sources: chat: {error:#}"));
|
||||
}
|
||||
},
|
||||
}
|
||||
u8::from(!clean)
|
||||
}
|
||||
|
||||
/// The longest span one request may name.
|
||||
const PURGE_SPAN_DAYS: i64 = 366;
|
||||
const REQUEST_BYTES_MAX: usize = 120;
|
||||
|
||||
/// A steward's purge request as the admin typed it, checked. Each error names its option.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Purge {
|
||||
pub room_id: String,
|
||||
pub from: String,
|
||||
pub to: String,
|
||||
pub request: String,
|
||||
pub window: PurgeWindow,
|
||||
}
|
||||
|
||||
fn instant(option: &str, text: &str) -> Result<DateTime<Utc>> {
|
||||
ensure!(text.ends_with('Z'), "--{option} must be an RFC 3339 UTC time ending in Z, got {text:?}");
|
||||
let at = DateTime::parse_from_rfc3339(text).map_err(|_| anyhow::anyhow!("--{option} must be an RFC 3339 UTC time ending in Z, got {text:?}"))?;
|
||||
Ok(at.with_timezone(&Utc))
|
||||
}
|
||||
|
||||
impl Purge {
|
||||
pub fn check(room: &str, from: &str, to: &str, request: &str, now: DateTime<Utc>) -> Result<Self> {
|
||||
ensure!(room.starts_with('!'), "--room must be a room id beginning with !, got {room:?}: an alias needs Matrix, and the journal prints both #alias (!id)");
|
||||
let (start, end) = (instant("from", from)?, instant("to", to)?);
|
||||
ensure!(start < end, "--from must be before --to");
|
||||
ensure!(end <= now, "--to must not be after now");
|
||||
ensure!(end - start <= chrono::Duration::days(PURGE_SPAN_DAYS), "the window must be at most {PURGE_SPAN_DAYS} days");
|
||||
ensure!(!request.is_empty() && request.len() <= REQUEST_BYTES_MAX, "--request must be 1 to {REQUEST_BYTES_MAX} bytes");
|
||||
let ms = |at: DateTime<Utc>| u64::try_from(at.timestamp_millis()).context("a purge time before 1970");
|
||||
Ok(Self { room_id: room.to_owned(), from: from.to_owned(), to: to.to_owned(), request: request.to_owned(), window: PurgeWindow { from_ms: ms(start)?, to_ms: ms(end)? } })
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes the tombstone and returns the line the admin sees; the free-text request is quoted and escaped so it cannot forge a journal line. Touches no point and makes no Matrix call;
|
||||
/// the caller holds the job's lock. The next run applies it.
|
||||
pub fn purge<S: SourceIndex>(index: &mut S, purge: &Purge) -> Result<String> {
|
||||
request_purge(index, &purge.room_id, purge.window)?;
|
||||
let DescribedSources::Labelled { cursors, .. } = index.describe_sources(INDEX_CALL_TIMEOUT).context("describing the source collection")? else {
|
||||
bail!("the source collection is absent or not Huginn's");
|
||||
};
|
||||
let windows = cursors.get(&purge_key(&purge.room_id)).map(|held| parse_windows(held)).transpose()?.map_or(0, |held| held.len());
|
||||
Ok(format!("chat-purge requested: {} {}..{} request={:?} windows={windows}", purge.room_id, purge.from, purge.to, purge.request))
|
||||
}
|
||||
287
crates/huginn-daemon/src/sources/job_tests.rs
Normal file
287
crates/huginn-daemon/src/sources/job_tests.rs
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
//! One run of the job: forge refresh and chat refresh over a fake Synapse reached through the real client,
|
||||
//! a fake vector store and the fake embedder; the exit status; the chat-off retirement; the purge request.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use anyhow::{Result, bail};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde_json::json;
|
||||
|
||||
use super::chat::{CHAT_PURGE_PREFIX, RoomHistory};
|
||||
use super::job::{Chat, Purge, purge, run};
|
||||
use super::matrix::HttpMatrix;
|
||||
use super::matrix_tests::{Room, Rooms, TOKEN, message, state, synapse};
|
||||
use super::room_scope::Estate;
|
||||
use super::tests::{FakeForge, Remote};
|
||||
use super::test_http::forge_of;
|
||||
use super::*;
|
||||
use crate::index::fakes::{FakeEmbedder, FakeSources};
|
||||
|
||||
const DAY0: u64 = 1_794_268_800_000; // 2026-11-10T00:00:00Z
|
||||
|
||||
fn now() -> DateTime<Utc> {
|
||||
DateTime::parse_from_rfc3339("2026-11-17T00:00:00Z").unwrap().with_timezone(&Utc)
|
||||
}
|
||||
|
||||
fn estate() -> Estate {
|
||||
Estate::new(["!e1:s", "!e2:s", "!e3:s"].map(str::to_owned).into()).unwrap()
|
||||
}
|
||||
|
||||
struct Out {
|
||||
said: Vec<String>,
|
||||
complained: Vec<String>,
|
||||
status: u8,
|
||||
}
|
||||
|
||||
struct Rig {
|
||||
embedder: FakeEmbedder,
|
||||
index: FakeSources,
|
||||
forge: FakeForge,
|
||||
_remote: Remote,
|
||||
root: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl Rig {
|
||||
fn new() -> Self {
|
||||
let remote = Remote::new();
|
||||
remote.commit(&[("README.md", b"# Demo\n")], &[]);
|
||||
Self { embedder: FakeEmbedder::new("d1"), index: FakeSources::default(), forge: FakeForge::of(&[("Demo", &remote)]), _remote: remote, root: tempfile::tempdir().unwrap() }
|
||||
}
|
||||
|
||||
fn go<H: RoomHistory>(&mut self, history: Option<&mut H>) -> Out {
|
||||
let (mut said, mut complained) = (Vec::new(), Vec::new());
|
||||
let estate = estate();
|
||||
let chat = history.map(|history| Chat { history, estate: &estate });
|
||||
let status = run(&mut self.embedder, &mut self.index, &mut self.forge, self.root.path(), chat, &mut |line| said.push(line), &mut |line| complained.push(line));
|
||||
Out { said, complained, status }
|
||||
}
|
||||
|
||||
fn chat_bodies(&self) -> Vec<String> {
|
||||
let mut bodies: Vec<String> = self.index.state.lock().unwrap().chat.values().flat_map(|point| point.payload.messages.iter().map(|m| m.body.clone())).collect();
|
||||
bodies.sort();
|
||||
bodies
|
||||
}
|
||||
|
||||
fn chat_rooms(&self) -> std::collections::BTreeSet<String> {
|
||||
self.index.state.lock().unwrap().chat.values().map(|point| point.payload.room_id.clone()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn rooms(list: Vec<(&str, Vec<serde_json::Value>, Vec<serde_json::Value>)>) -> Rooms {
|
||||
Arc::new(Mutex::new(list.into_iter().map(|(id, state, timeline)| (id.to_owned(), Room { state, timeline })).collect::<BTreeMap<_, _>>()))
|
||||
}
|
||||
|
||||
fn platform() -> String {
|
||||
["disc", "ord"].concat()
|
||||
}
|
||||
|
||||
fn world() -> Rooms {
|
||||
let relay = format!("@_{}_1:s", platform());
|
||||
rooms(vec![
|
||||
("!swarm:s", vec![state("m.room.canonical_alias", json!({ "alias": "#swarm:gamecult.org" })), state("m.room.join_rules", json!({ "join_rule": "invite" }))], vec![
|
||||
message("$m1", "@a:s", DAY0 - 5000, "first words"),
|
||||
message("$m2", "@b:s", DAY0 - 4000, "to be redacted"),
|
||||
message("$m3", "@a:s", DAY0 + 1000, "inside the window"),
|
||||
]),
|
||||
("!bridged:s", vec![state("m.bridge", json!({}))], vec![message("$b1", "@a:s", DAY0, "bridged talk")]),
|
||||
("!relay:s", vec![], vec![message("$d1", &relay, DAY0, "relayed talk")]),
|
||||
("!aliased:s", vec![state("m.room.canonical_alias", json!({ "alias": "#Users:gamecult.org" }))], vec![message("$u1", "@a:s", DAY0, "estate talk")]),
|
||||
])
|
||||
}
|
||||
|
||||
fn all_lines(out: &Out) -> String {
|
||||
out.said.iter().chain(&out.complained).cloned().collect::<Vec<_>>().join("\n")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_full_run_over_a_fake_synapse() {
|
||||
let world = world();
|
||||
let (base, asked) = synapse(&world);
|
||||
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||
let mut rig = Rig::new();
|
||||
|
||||
let first = rig.go(Some(&mut matrix));
|
||||
assert_eq!(first.status, 1, "joined bridged, relay and estate rooms are refused: {:?}", first.complained);
|
||||
assert_eq!(rig.chat_rooms(), ["!swarm:s".to_owned()].into(), "an invite-only room is ingested; nothing of the refused rooms is");
|
||||
assert_eq!(rig.chat_bodies(), ["first words", "inside the window", "to be redacted"]);
|
||||
for refused in ["!bridged:s", "!relay:s", "!aliased:s"] {
|
||||
assert!(first.complained.iter().any(|line| line.starts_with("refused:") && line.contains(refused)), "{refused}: {:?}", first.complained);
|
||||
}
|
||||
assert!(!rig.embedder.texts().iter().any(|text| text.contains("bridged talk") || text.contains("relayed talk") || text.contains("estate talk")));
|
||||
assert!(first.said.iter().any(|line| line.starts_with("Demo ")), "the forge's line is journaled too");
|
||||
assert!(first.said.iter().any(|line| line.starts_with("#swarm:gamecult.org (!swarm:s)") && line.contains("events=3")), "{:?}", first.said);
|
||||
assert!(asked.seen().iter().all(|seen| seen.method == "GET"));
|
||||
|
||||
// The steward asks for the window; then the room edits, redacts and leaks a key.
|
||||
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "follow_up:x", now()).unwrap();
|
||||
assert_eq!(purge(&mut rig.index, &request).unwrap(), "chat-purge requested: !swarm:s 2026-11-10T00:00:00Z..2026-11-11T00:00:00Z request=\"follow_up:x\" windows=1");
|
||||
let before = rig.index.cursors();
|
||||
assert_eq!(purge(&mut rig.index, &request).unwrap().ends_with("windows=1"), true);
|
||||
assert_eq!(rig.index.cursors(), before, "a repeated request writes nothing");
|
||||
let secret = format!("key ghp_{}", "k2L9".repeat(9));
|
||||
{
|
||||
let mut held = world.lock().unwrap();
|
||||
let room = held.get_mut("!swarm:s").unwrap();
|
||||
room.timeline.push(json!({ "type": "m.room.message", "event_id": "$edit", "sender": "@a:s", "origin_server_ts": DAY0 + 90_000_000, "content": {
|
||||
"msgtype": "m.text", "body": "* first words, amended", "m.new_content": { "msgtype": "m.text", "body": "first words, amended" }, "m.relates_to": { "rel_type": "m.replace", "event_id": "$m1" } } }));
|
||||
room.timeline.push(json!({ "type": "m.room.redaction", "event_id": "$red", "sender": "@b:s", "origin_server_ts": DAY0 + 91_000_000, "content": { "redacts": "$m2" } }));
|
||||
room.timeline.push(message("$m4", "@a:s", DAY0 + 92_000_000, &secret));
|
||||
}
|
||||
let second = rig.go(Some(&mut matrix));
|
||||
assert_eq!(rig.chat_bodies().len(), 2, "{:?}", rig.chat_bodies());
|
||||
let bodies = rig.chat_bodies().join("\n");
|
||||
assert!(bodies.contains("first words, amended") && !bodies.contains("to be redacted") && !bodies.contains("inside the window"), "{bodies}");
|
||||
assert!(!bodies.contains("ghp_k2L9") && !rig.embedder.texts().iter().any(|text| text.contains("ghp_k2L9")), "the key was scrubbed before it was embedded");
|
||||
let text = all_lines(&second);
|
||||
assert!(second.said.iter().any(|line| line.contains("purged=")), "{text}");
|
||||
assert!(second.complained.iter().any(|line| line == "scrubbed: !swarm:s $m4 key-shape" || line.starts_with("scrubbed: !swarm:s $m4 ")), "{text}");
|
||||
for body in ["first words", "to be redacted", "inside the window", "relayed talk", "ghp_"] {
|
||||
assert!(!text.contains(body), "the journal carries ids and counts, not bodies: {body}");
|
||||
}
|
||||
let third = rig.go(Some(&mut matrix));
|
||||
assert!(!rig.chat_bodies().join("\n").contains("inside the window"), "a purged window does not come back");
|
||||
assert!(third.complained.iter().all(|line| line.starts_with("refused:")), "{:?}", third.complained);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_chat_flags_purge_the_corpus_and_make_no_matrix_call() {
|
||||
let world = world();
|
||||
let (base, _) = synapse(&world);
|
||||
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||
let mut rig = Rig::new();
|
||||
rig.go(Some(&mut matrix));
|
||||
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "r", now()).unwrap();
|
||||
purge(&mut rig.index, &request).unwrap();
|
||||
assert!(!rig.chat_bodies().is_empty());
|
||||
|
||||
let (hung, asked) = forge_of(|_| None::<String>);
|
||||
drop(hung);
|
||||
let out = rig.go::<HttpMatrix>(None);
|
||||
assert_eq!(out.status, 0, "{:?}", out.complained);
|
||||
assert!(rig.chat_bodies().is_empty() && rig.chat_rooms().is_empty());
|
||||
assert!(rig.index.cursors().keys().any(|key| key.starts_with(CHAT_PURGE_PREFIX)), "tombstones stay");
|
||||
assert!(!rig.index.cursors().keys().any(|key| key.starts_with("chat:")), "cursors go");
|
||||
assert!(asked.seen().is_empty());
|
||||
}
|
||||
|
||||
struct BrokenForge;
|
||||
|
||||
impl Forge for BrokenForge {
|
||||
fn catalog(&mut self) -> Result<Catalog> {
|
||||
bail!("the forge is down")
|
||||
}
|
||||
|
||||
fn clone_url(&self, _name: &str) -> String {
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
struct DeadSynapse;
|
||||
|
||||
impl RoomHistory for DeadSynapse {
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||
bail!("synapse is down")
|
||||
}
|
||||
|
||||
fn facts(&mut self, _room_id: &str) -> Result<super::room_scope::RoomFacts> {
|
||||
bail!("synapse is down")
|
||||
}
|
||||
|
||||
fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result<Option<super::chat::EventPage>> {
|
||||
bail!("synapse is down")
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exit_status_is_clean_only_when_forge_and_chat_both_are() {
|
||||
let clean = rooms(vec![("!swarm:s", vec![], vec![message("$m1", "@a:s", 1, "words")])]);
|
||||
let (base, _) = synapse(&clean);
|
||||
let mut rig = Rig::new();
|
||||
let out = rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
|
||||
assert_eq!((out.status, out.complained.len()), (0, 0));
|
||||
|
||||
// Scrub hits alone are not a failure; they are reported.
|
||||
clean.lock().unwrap().get_mut("!swarm:s").unwrap().timeline.push(message("$m2", "@a:s", 2, &format!("ghp_{}", "k2L9".repeat(9))));
|
||||
let out = rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
|
||||
assert_eq!(out.status, 0, "{:?}", out.complained);
|
||||
assert!(out.complained.iter().any(|line| line.starts_with("scrubbed: !swarm:s $m2")));
|
||||
|
||||
// A chat failure still reports the forge, and a forge failure still runs chat.
|
||||
let out = rig.go(Some(&mut DeadSynapse));
|
||||
assert_eq!(out.status, 1);
|
||||
assert!(out.complained.iter().any(|line| line.contains("synapse is down")));
|
||||
let (mut said, mut complained) = (Vec::new(), Vec::new());
|
||||
let (estate, mut ok) = (estate(), HttpMatrix::new(&base, TOKEN));
|
||||
let status = run(&mut rig.embedder, &mut rig.index, &mut BrokenForge, rig.root.path(), Some(Chat { history: &mut ok, estate: &estate }), &mut |l| said.push(l), &mut |l| complained.push(l));
|
||||
assert_eq!(status, 1);
|
||||
assert!(complained.iter().any(|line| line.contains("the forge is down")), "{complained:?}");
|
||||
assert!(said.iter().any(|line| line.contains("(!swarm:s)")), "chat ran after the forge failed: {said:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_cannot_forge_a_journal_line() {
|
||||
let mut rig = Rig::new();
|
||||
rig.go::<HttpMatrix>(None);
|
||||
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "x\nrefused: !forged:s", now()).unwrap();
|
||||
let line = purge(&mut rig.index, &request).unwrap();
|
||||
assert_eq!(line.lines().count(), 1, "{line:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_room_report_goes_to_the_complaint_stream_and_fails_the_run() {
|
||||
let mut rig = Rig::new();
|
||||
struct FactsDown(Rooms);
|
||||
impl RoomHistory for FactsDown {
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||
Ok(vec!["!swarm:s".to_owned()])
|
||||
}
|
||||
fn facts(&mut self, _room_id: &str) -> Result<super::room_scope::RoomFacts> {
|
||||
bail!("facts are down")
|
||||
}
|
||||
fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result<Option<super::chat::EventPage>> {
|
||||
bail!("unreachable")
|
||||
}
|
||||
}
|
||||
let out = rig.go(Some(&mut FactsDown(rooms(vec![]))));
|
||||
assert_eq!(out.status, 1);
|
||||
assert!(out.complained.iter().any(|line| line.contains("!swarm:s") && line.contains("facts are down")), "{:?}", out.complained);
|
||||
assert!(!out.said.iter().any(|line| line.contains("facts are down")), "{:?}", out.said);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_purge_makes_no_matrix_request() {
|
||||
let world = world();
|
||||
let (base, asked) = synapse(&world);
|
||||
let mut rig = Rig::new();
|
||||
rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
|
||||
let made = asked.seen().len();
|
||||
let request = Purge::check("!swarm:s", "2026-11-10T00:00:00Z", "2026-11-11T00:00:00Z", "r", now()).unwrap();
|
||||
purge(&mut rig.index, &request).unwrap();
|
||||
assert_eq!(asked.seen().len(), made);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_retirement_failure_goes_to_the_complaint_stream() {
|
||||
struct Left;
|
||||
impl RoomHistory for Left {
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
fn facts(&mut self, _room_id: &str) -> Result<super::room_scope::RoomFacts> {
|
||||
bail!("unreachable")
|
||||
}
|
||||
fn events_after(&mut self, _room_id: &str, _after: Option<&str>, _limit: u32) -> Result<Option<super::chat::EventPage>> {
|
||||
bail!("unreachable")
|
||||
}
|
||||
}
|
||||
let world = rooms(vec![("!swarm:s", vec![], vec![message("$m1", "@a:s", 1, "words")])]);
|
||||
let (base, _) = synapse(&world);
|
||||
let mut rig = Rig::new();
|
||||
rig.go(Some(&mut HttpMatrix::new(&base, TOKEN)));
|
||||
rig.index.state.lock().unwrap().fail_chat_deletes = true;
|
||||
let out = rig.go(Some(&mut Left));
|
||||
assert_eq!(out.status, 1);
|
||||
assert!(out.complained.iter().any(|line| line.starts_with("huginn-sources: retiring !swarm:s failed")), "{:?}", out.complained);
|
||||
assert!(!out.said.iter().any(|line| line.contains("retiring")), "{:?}", out.said);
|
||||
}
|
||||
159
crates/huginn-daemon/src/sources/matrix.rs
Normal file
159
crates/huginn-daemon/src/sources/matrix.rs
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
//! Synapse's client API as the reader user sees it: Matrix JSON in, `RoomEvent`
|
||||
//! and `RoomFacts` out. JSON for Matrix is spoken here and nowhere else. This
|
||||
//! client reads with GET only and holds no policy: it filters nothing, decides
|
||||
//! no room's or sender's scope, and names no platform. What is in scope is
|
||||
//! `room_scope`'s; the token is read at one place, the `Authorization` header.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::chat::{EventKind, EventPage, RoomEvent, RoomHistory};
|
||||
use super::room_scope::RoomFacts;
|
||||
|
||||
const CALL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// A bridge's state event types: either marks a room as relaying another platform.
|
||||
const BRIDGE_TYPES: [&str; 2] = ["m.bridge", "uk.half-shot.bridge"];
|
||||
|
||||
pub struct HttpMatrix {
|
||||
agent: ureq::Agent,
|
||||
base: String,
|
||||
token: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for HttpMatrix {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.debug_struct("HttpMatrix").field("base", &self.base).finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
/// A path segment with everything but unreserved characters escaped.
|
||||
fn segment(raw: &str) -> String {
|
||||
let mut out = String::with_capacity(raw.len());
|
||||
for byte in raw.bytes() {
|
||||
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
|
||||
out.push(char::from(byte));
|
||||
} else {
|
||||
out.push_str(&format!("%{byte:02X}"));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
impl HttpMatrix {
|
||||
pub fn new(base_url: &str, token: &str) -> Self {
|
||||
let agent: ureq::Agent =
|
||||
ureq::Agent::config_builder().timeout_global(Some(CALL_TIMEOUT)).http_status_as_error(false).build().into();
|
||||
Self { agent, base: base_url.trim_end_matches('/').to_owned(), token: token.to_owned() }
|
||||
}
|
||||
|
||||
/// One GET of `path` (already escaped): the status and the JSON body. No message made here carries the token.
|
||||
fn get(&self, path: &str) -> Result<(u16, Value)> {
|
||||
let url = format!("{}{path}", self.base);
|
||||
let mut response = self
|
||||
.agent
|
||||
.get(&url)
|
||||
.header("Authorization", format!("Bearer {}", self.token))
|
||||
.call()
|
||||
.with_context(|| format!("no answer from {url}"))?;
|
||||
let status = response.status().as_u16();
|
||||
let body = response.body_mut().read_to_string().with_context(|| format!("reading the answer from {url}"))?;
|
||||
Ok((status, serde_json::from_str(&body).unwrap_or(Value::Null)))
|
||||
}
|
||||
|
||||
fn ok(&self, path: &str) -> Result<Value> {
|
||||
let (status, body) = self.get(path)?;
|
||||
if status != 200 {
|
||||
bail!("synapse answered {status} for {path}");
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
}
|
||||
|
||||
fn event_of(raw: &Value) -> Result<RoomEvent> {
|
||||
let text = |key: &str| raw[key].as_str().map(str::to_owned).with_context(|| format!("an event has no {key}"));
|
||||
let content = &raw["content"];
|
||||
let content_keys = content.as_object().map(|map| map.keys().cloned().collect()).unwrap_or_default();
|
||||
Ok(RoomEvent { event_id: text("event_id")?, sender: text("sender")?, ts_ms: raw["origin_server_ts"].as_u64().context("an event has no origin_server_ts")?, content_keys, kind: kind_of(raw) })
|
||||
}
|
||||
|
||||
fn kind_of(raw: &Value) -> EventKind {
|
||||
let content = &raw["content"];
|
||||
if !raw["unsigned"]["redacted_because"].is_null() {
|
||||
return EventKind::Other;
|
||||
}
|
||||
match raw["type"].as_str() {
|
||||
Some("m.room.redaction") => match content["redacts"].as_str().or_else(|| raw["redacts"].as_str()) {
|
||||
Some(target) => EventKind::Redaction { target: target.to_owned() },
|
||||
None => EventKind::Other,
|
||||
},
|
||||
Some("m.room.message") if matches!(content["msgtype"].as_str(), Some("m.text" | "m.notice" | "m.emote")) => {
|
||||
let Some(body) = content["body"].as_str() else { return EventKind::Other };
|
||||
let relation = &content["m.relates_to"];
|
||||
if relation["rel_type"].as_str() == Some("m.replace") {
|
||||
return match (relation["event_id"].as_str(), content["m.new_content"]["body"].as_str()) {
|
||||
(Some(target), Some(new)) => EventKind::Edit { target: target.to_owned(), body: new.to_owned() },
|
||||
_ => EventKind::Other,
|
||||
};
|
||||
}
|
||||
EventKind::Message { body: body.to_owned() }
|
||||
}
|
||||
_ => EventKind::Other,
|
||||
}
|
||||
}
|
||||
|
||||
impl RoomHistory for HttpMatrix {
|
||||
fn joined_rooms(&mut self) -> Result<Vec<String>> {
|
||||
let body = self.ok("/_matrix/client/v3/joined_rooms")?;
|
||||
let rooms = body["joined_rooms"].as_array().context("the joined rooms answer is not a list")?;
|
||||
Ok(rooms.iter().filter_map(|room| room.as_str().map(str::to_owned)).collect())
|
||||
}
|
||||
|
||||
fn facts(&mut self, room_id: &str) -> Result<RoomFacts> {
|
||||
let body = self.ok(&format!("/_matrix/client/v3/rooms/{}/state", segment(room_id)))?;
|
||||
let state = body.as_array().context("the room state is not a list")?;
|
||||
let mut facts = RoomFacts { room_id: room_id.to_owned(), alias: None, alt_aliases: Vec::new(), name: None, encrypted: false, direct: false, bridge: None };
|
||||
for event in state {
|
||||
let content = &event["content"];
|
||||
match event["type"].as_str() {
|
||||
Some("m.room.canonical_alias") => {
|
||||
facts.alias = content["alias"].as_str().map(str::to_owned);
|
||||
facts.alt_aliases = content["alt_aliases"].as_array().map(|list| list.iter().filter_map(|alias| alias.as_str().map(str::to_owned)).collect()).unwrap_or_default();
|
||||
}
|
||||
Some("m.room.name") => facts.name = content["name"].as_str().map(str::to_owned),
|
||||
Some("m.room.encryption") => facts.encrypted = true,
|
||||
Some("m.room.member") if content["is_direct"].as_bool() == Some(true) => facts.direct = true,
|
||||
Some(kind) if BRIDGE_TYPES.contains(&kind) => {
|
||||
facts.bridge.get_or_insert_with(|| kind.to_owned());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(facts)
|
||||
}
|
||||
|
||||
fn events_after(&mut self, room_id: &str, after: Option<&str>, limit: u32) -> Result<Option<EventPage>> {
|
||||
let room = segment(room_id);
|
||||
let from = match after {
|
||||
None => String::new(),
|
||||
Some(event) => {
|
||||
let (status, body) = self.get(&format!("/_matrix/client/v3/rooms/{room}/context/{}?limit=0", segment(event)))?;
|
||||
match status {
|
||||
200 => match body["end"].as_str() {
|
||||
Some(end) => format!("&from={}", segment(end)),
|
||||
None => bail!("synapse gave no end token for the context of {event} in {room_id}"),
|
||||
},
|
||||
404 => return Ok(None),
|
||||
other => bail!("synapse answered {other} for the context of {event} in {room_id}"),
|
||||
}
|
||||
}
|
||||
};
|
||||
let body = self.ok(&format!("/_matrix/client/v3/rooms/{room}/messages?dir=f&limit={limit}{from}"))?;
|
||||
let chunk = body["chunk"].as_array().context("the messages answer has no chunk")?;
|
||||
let events = chunk.iter().map(event_of).collect::<Result<Vec<_>>>()?;
|
||||
let more = body["end"].as_str().is_some() && !events.is_empty();
|
||||
Ok(Some(EventPage { events, more }))
|
||||
}
|
||||
}
|
||||
207
crates/huginn-daemon/src/sources/matrix_tests.rs
Normal file
207
crates/huginn-daemon/src/sources/matrix_tests.rs
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
//! The Matrix client against a fake Synapse that speaks the client API's JSON.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::chat::{EventKind, RoomHistory};
|
||||
use super::matrix::HttpMatrix;
|
||||
use super::test_http::{Asked, Reply, forge_of};
|
||||
|
||||
pub(super) const TOKEN: &str = "syt_reader_9f8e7d6c5b4a";
|
||||
|
||||
/// A room as the fake server holds it: its state events and its timeline.
|
||||
pub(super) struct Room {
|
||||
pub state: Vec<Value>,
|
||||
pub timeline: Vec<Value>,
|
||||
}
|
||||
|
||||
pub(super) type Rooms = Arc<Mutex<BTreeMap<String, Room>>>;
|
||||
|
||||
pub(super) fn decode(raw: &str) -> String {
|
||||
let bytes = raw.as_bytes();
|
||||
let mut out = Vec::new();
|
||||
let mut at = 0;
|
||||
while at < bytes.len() {
|
||||
if bytes[at] == b'%' {
|
||||
out.push(u8::from_str_radix(&raw[at + 1..at + 3], 16).unwrap());
|
||||
at += 3;
|
||||
} else {
|
||||
out.push(bytes[at]);
|
||||
at += 1;
|
||||
}
|
||||
}
|
||||
String::from_utf8(out).unwrap()
|
||||
}
|
||||
|
||||
pub(super) fn message(id: &str, sender: &str, ts: u64, body: &str) -> Value {
|
||||
json!({ "type": "m.room.message", "event_id": id, "sender": sender, "origin_server_ts": ts, "content": { "msgtype": "m.text", "body": body } })
|
||||
}
|
||||
|
||||
pub(super) fn state(kind: &str, content: Value) -> Value {
|
||||
json!({ "type": kind, "state_key": "", "content": content })
|
||||
}
|
||||
|
||||
/// A Synapse over `rooms`, joined to the rooms in the map: GET only, bearer required.
|
||||
pub(super) fn synapse(rooms: &Rooms) -> (String, Asked) {
|
||||
let rooms = Arc::clone(rooms);
|
||||
forge_of(move |raw| {
|
||||
let (path, query) = raw.split_once('?').unwrap_or((raw, ""));
|
||||
let path = decode(path);
|
||||
let rooms = rooms.lock().unwrap();
|
||||
let rest = path.strip_prefix("/_matrix/client/v3/")?;
|
||||
if rest == "joined_rooms" {
|
||||
return Some(Reply::from(json!({ "joined_rooms": rooms.keys().collect::<Vec<_>>() }).to_string()));
|
||||
}
|
||||
let rest = rest.strip_prefix("rooms/")?;
|
||||
let (id, rest) = rest.split_once('/')?;
|
||||
let Some(room) = rooms.get(id) else { return Some(Reply::status(404, r#"{"errcode":"M_NOT_FOUND"}"#)) };
|
||||
let query: BTreeMap<String, String> = query.split('&').filter_map(|pair| pair.split_once('=')).map(|(k, v)| (k.to_owned(), decode(v))).collect();
|
||||
if rest == "state" {
|
||||
return Some(Reply::from(Value::Array(room.state.clone()).to_string()));
|
||||
}
|
||||
if let Some(event) = rest.strip_prefix("context/") {
|
||||
return Some(match room.timeline.iter().any(|held| held["event_id"] == event) {
|
||||
true => Reply::from(json!({ "end": format!("t_{event}"), "events_before": [], "events_after": [] }).to_string()),
|
||||
false => Reply::status(404, r#"{"errcode":"M_NOT_FOUND"}"#),
|
||||
});
|
||||
}
|
||||
if rest == "messages" {
|
||||
let start = match query.get("from") {
|
||||
Some(from) => 1 + room.timeline.iter().position(|held| format!("t_{}", held["event_id"].as_str().unwrap()) == *from)?,
|
||||
None => 0,
|
||||
};
|
||||
let limit: usize = query["limit"].parse().unwrap();
|
||||
let chunk: Vec<&Value> = room.timeline.iter().skip(start).take(limit).collect();
|
||||
let end = chunk.last().map(|last| format!("t_{}", last["event_id"].as_str().unwrap()));
|
||||
return Some(Reply::from(json!({ "chunk": chunk, "end": end }).to_string()));
|
||||
}
|
||||
None
|
||||
})
|
||||
}
|
||||
|
||||
fn one_room(state: Vec<Value>, timeline: Vec<Value>) -> Rooms {
|
||||
Arc::new(Mutex::new(BTreeMap::from([("!r:s".to_owned(), Room { state, timeline })])))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn state_maps_to_facts_and_the_join_rule_is_not_read() {
|
||||
let plain = one_room(vec![state("m.room.join_rules", json!({ "join_rule": "invite" }))], vec![]);
|
||||
let (base, _) = synapse(&plain);
|
||||
let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap();
|
||||
assert_eq!((facts.alias, facts.name, facts.encrypted, facts.direct, facts.bridge), (None, None, false, false, None));
|
||||
for rule in ["public", "knock", "invite"] {
|
||||
let rooms = one_room(vec![state("m.room.join_rules", json!({ "join_rule": rule }))], vec![]);
|
||||
let (base, _) = synapse(&rooms);
|
||||
let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap();
|
||||
assert!(!facts.direct && !facts.encrypted && facts.bridge.is_none(), "{rule}");
|
||||
}
|
||||
let full = one_room(
|
||||
vec![
|
||||
state("m.room.canonical_alias", json!({ "alias": "#a:s", "alt_aliases": ["#b:s", "#c:s"] })),
|
||||
state("m.room.name", json!({ "name": "Swarm" })),
|
||||
state("m.room.encryption", json!({ "algorithm": "m.megolm.v1.aes-sha2" })),
|
||||
json!({ "type": "m.room.member", "state_key": "@x:s", "content": { "membership": "join" } }),
|
||||
json!({ "type": "m.room.member", "state_key": "@y:s", "content": { "membership": "join", "is_direct": true } }),
|
||||
state("uk.half-shot.bridge", json!({})),
|
||||
],
|
||||
vec![],
|
||||
);
|
||||
let (base, _) = synapse(&full);
|
||||
let facts = HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap();
|
||||
assert_eq!(facts.alias.as_deref(), Some("#a:s"));
|
||||
assert_eq!(facts.alt_aliases, ["#b:s", "#c:s"]);
|
||||
assert_eq!(facts.name.as_deref(), Some("Swarm"));
|
||||
assert!(facts.encrypted && facts.direct);
|
||||
assert_eq!(facts.bridge.as_deref(), Some("uk.half-shot.bridge"));
|
||||
let relay = one_room(vec![state("m.bridge", json!({}))], vec![]);
|
||||
let (base, _) = synapse(&relay);
|
||||
assert_eq!(HttpMatrix::new(&base, TOKEN).facts("!r:s").unwrap().bridge.as_deref(), Some("m.bridge"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn events_map_to_kinds_and_content_keys_are_names_only() {
|
||||
let mut other = message("$img", "@a:s", 3, "pic.png");
|
||||
other["content"]["msgtype"] = json!("m.image");
|
||||
let mut pruned = message("$pruned", "@a:s", 4, "was here");
|
||||
pruned["unsigned"] = json!({ "redacted_because": { "event_id": "$x" } });
|
||||
let mut notice = message("$n", "@a:s", 5, "n");
|
||||
notice["content"]["msgtype"] = json!("m.notice");
|
||||
let mut emote = message("$e", "@a:s", 6, "e");
|
||||
emote["content"]["msgtype"] = json!("m.emote");
|
||||
let edit = json!({ "type": "m.room.message", "event_id": "$edit", "sender": "@a:s", "origin_server_ts": 7, "content": {
|
||||
"msgtype": "m.text", "body": "* new", "m.new_content": { "msgtype": "m.text", "body": "new" }, "m.relates_to": { "rel_type": "m.replace", "event_id": "$m1" } } });
|
||||
let redaction = json!({ "type": "m.room.redaction", "event_id": "$red", "sender": "@a:s", "origin_server_ts": 8, "content": { "redacts": "$m1" } });
|
||||
let old_redaction = json!({ "type": "m.room.redaction", "event_id": "$red2", "sender": "@a:s", "origin_server_ts": 9, "redacts": "$m2", "content": {} });
|
||||
let encrypted = json!({ "type": "m.room.encrypted", "event_id": "$enc", "sender": "@a:s", "origin_server_ts": 10, "content": { "ciphertext": "xx" } });
|
||||
let timeline = vec![message("$m1", "@a:s", 1, "hello"), other, pruned, notice, emote, edit, redaction, old_redaction, encrypted];
|
||||
let rooms = one_room(vec![], timeline);
|
||||
let (base, _) = synapse(&rooms);
|
||||
let page = HttpMatrix::new(&base, TOKEN).events_after("!r:s", None, 50).unwrap().unwrap();
|
||||
let kinds: Vec<EventKind> = page.events.iter().map(|event| event.kind.clone()).collect();
|
||||
assert_eq!(
|
||||
kinds,
|
||||
[
|
||||
EventKind::Message { body: "hello".into() },
|
||||
EventKind::Other,
|
||||
EventKind::Other,
|
||||
EventKind::Message { body: "n".into() },
|
||||
EventKind::Message { body: "e".into() },
|
||||
EventKind::Edit { target: "$m1".into(), body: "new".into() },
|
||||
EventKind::Redaction { target: "$m1".into() },
|
||||
EventKind::Redaction { target: "$m2".into() },
|
||||
EventKind::Other,
|
||||
]
|
||||
);
|
||||
assert_eq!(page.events[0].content_keys, ["body", "msgtype"]);
|
||||
assert_eq!(page.events[5].content_keys, ["body", "m.new_content", "m.relates_to", "msgtype"], "names, never values");
|
||||
assert_eq!((page.events[0].event_id.as_str(), page.events[0].sender.as_str(), page.events[0].ts_ms), ("$m1", "@a:s", 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_uses_the_cursor_events_end_token_and_a_missing_event_is_none() {
|
||||
let rooms = one_room(vec![], vec![message("$1", "@a:s", 1, "a"), message("$2", "@a:s", 2, "b"), message("$3", "@a:s", 3, "c")]);
|
||||
let (base, asked) = synapse(&rooms);
|
||||
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||
let page = matrix.events_after("!r:s", Some("$1"), 1).unwrap().unwrap();
|
||||
assert_eq!(page.events.iter().map(|event| event.event_id.as_str()).collect::<Vec<_>>(), ["$2"]);
|
||||
assert!(page.more);
|
||||
assert_eq!(asked.paths(), ["/_matrix/client/v3/rooms/%21r%3As/context/%241?limit=0", "/_matrix/client/v3/rooms/%21r%3As/messages?dir=f&limit=1&from=t_%241"]);
|
||||
assert!(matrix.events_after("!r:s", Some("$unknown"), 5).unwrap().is_none());
|
||||
let tail = matrix.events_after("!r:s", Some("$3"), 5).unwrap().unwrap();
|
||||
assert!(tail.events.is_empty() && !tail.more, "an empty chunk ends the run");
|
||||
let first = matrix.events_after("!r:s", None, 2).unwrap().unwrap();
|
||||
assert_eq!(first.events.len(), 2);
|
||||
assert_eq!(asked.paths().last().unwrap(), "/_matrix/client/v3/rooms/%21r%3As/messages?dir=f&limit=2", "no from, no context call");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_get_always_the_bearer_and_the_token_is_never_printed() {
|
||||
let rooms = one_room(vec![], vec![message("$1", "@a:s", 1, "a")]);
|
||||
let (base, asked) = synapse(&rooms);
|
||||
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||
matrix.joined_rooms().unwrap();
|
||||
matrix.facts("!r:s").unwrap();
|
||||
matrix.events_after("!r:s", Some("$1"), 3).unwrap();
|
||||
let seen = asked.seen();
|
||||
assert!(seen.len() >= 4 && seen.iter().all(|seen| seen.method == "GET" && seen.authorization.as_deref() == Some(&format!("Bearer {TOKEN}"))), "{seen:?}");
|
||||
let (base, _) = forge_of(|_| Some(Reply::status(401, &format!(r#"{{"errcode":"M_UNKNOWN_TOKEN","error":"bad {TOKEN}"}}"#))));
|
||||
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||
let error = matrix.joined_rooms().unwrap_err();
|
||||
assert!(format!("{error:#}").contains("401"));
|
||||
assert!(!format!("{error:#}{error:?}").contains(TOKEN));
|
||||
assert!(!format!("{matrix:?}").contains(TOKEN));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ids_are_percent_encoded_in_paths() {
|
||||
let weird = "$a+b/c=d:s";
|
||||
let rooms: Rooms = Arc::new(Mutex::new(BTreeMap::from([("!ro om:s".to_owned(), Room { state: vec![], timeline: vec![message(weird, "@a:s", 1, "x"), message("$2", "@a:s", 2, "y")] })])));
|
||||
let (base, asked) = synapse(&rooms);
|
||||
let mut matrix = HttpMatrix::new(&base, TOKEN);
|
||||
let page = matrix.events_after("!ro om:s", Some(weird), 5).unwrap().unwrap();
|
||||
assert_eq!(page.events.len(), 1, "the server decoded the room and event ids to the originals");
|
||||
let raw = asked.paths();
|
||||
assert!(raw.iter().all(|path| !path.contains('+') && !path.contains(' ') && !path.contains("!ro")), "{raw:?}");
|
||||
}
|
||||
|
|
@ -30,11 +30,20 @@ pub mod chat_purge;
|
|||
pub mod chunk;
|
||||
pub mod forge;
|
||||
pub mod git;
|
||||
pub mod job;
|
||||
pub mod matrix;
|
||||
pub mod room_scope;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_http;
|
||||
|
||||
#[cfg(test)]
|
||||
mod chat_tests;
|
||||
#[cfg(test)]
|
||||
mod job_tests;
|
||||
#[cfg(test)]
|
||||
mod matrix_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use std::collections::hash_map::Entry;
|
||||
|
|
|
|||
92
crates/huginn-daemon/src/sources/test_http.rs
Normal file
92
crates/huginn-daemon/src/sources/test_http.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
//! The one fake HTTP server the source tests talk to (the forge's and Matrix's):
|
||||
//! it answers by path, holds open a path it has no answer for as a hung server
|
||||
//! does, and records each request's method, path and `Authorization` header.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
/// What a fake server answers: a status, the `Location` header when there is one, and a body.
|
||||
pub struct Reply {
|
||||
pub status: u16,
|
||||
pub location: Option<String>,
|
||||
pub body: String,
|
||||
}
|
||||
|
||||
impl From<String> for Reply {
|
||||
fn from(body: String) -> Self {
|
||||
Self { status: 200, location: None, body }
|
||||
}
|
||||
}
|
||||
|
||||
impl Reply {
|
||||
/// What the live forge answers an anonymous reader for a private repo.
|
||||
pub fn not_found() -> Self {
|
||||
Self { status: 404, location: None, body: json!({ "message": "The target couldn't be found." }).to_string() }
|
||||
}
|
||||
|
||||
/// What the live forge answers for a renamed repo: a 301 to its public address, `/forge` prefix and all.
|
||||
pub fn moved_to(location: &str) -> Self {
|
||||
Self { status: 301, location: Some(location.to_owned()), body: String::new() }
|
||||
}
|
||||
|
||||
pub fn status(status: u16, body: &str) -> Self {
|
||||
Self { status, location: None, body: body.to_owned() }
|
||||
}
|
||||
}
|
||||
|
||||
/// One request as the server saw it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Seen {
|
||||
pub method: String,
|
||||
pub path: String,
|
||||
pub authorization: Option<String>,
|
||||
}
|
||||
|
||||
/// Every request a fake server has seen, in order.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct Asked(Arc<Mutex<Vec<Seen>>>);
|
||||
|
||||
impl Asked {
|
||||
pub fn seen(&self) -> Vec<Seen> {
|
||||
self.0.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
pub fn paths(&self) -> Vec<String> {
|
||||
self.0.lock().unwrap().iter().map(|seen| seen.path.clone()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Answers each request by its path from `routes`; a path it has no answer for is held open and never answered.
|
||||
pub fn forge_of<R: Into<Reply>>(routes: impl Fn(&str) -> Option<R> + Send + 'static) -> (String, Asked) {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let asked = Asked::default();
|
||||
let record = asked.clone();
|
||||
std::thread::spawn(move || {
|
||||
let mut held = Vec::new();
|
||||
for stream in listener.incoming() {
|
||||
let mut stream = stream.unwrap();
|
||||
let mut buffer = [0_u8; 4096];
|
||||
let read = stream.read(&mut buffer).unwrap();
|
||||
let request = String::from_utf8_lossy(&buffer[..read]).into_owned();
|
||||
let mut words = request.split_whitespace();
|
||||
let method = words.next().unwrap_or_default().to_owned();
|
||||
let path = words.next().unwrap_or_default().to_owned();
|
||||
let authorization = request
|
||||
.lines()
|
||||
.find_map(|line| line.split_once(':').filter(|(name, _)| name.eq_ignore_ascii_case("authorization")))
|
||||
.map(|(_, value)| value.trim().to_owned());
|
||||
record.0.lock().unwrap().push(Seen { method, path: path.clone(), authorization });
|
||||
match routes(&path).map(Into::into) {
|
||||
Some(Reply { status, location, body }) => {
|
||||
let location = location.map(|location| format!("location: {location}\r\n")).unwrap_or_default();
|
||||
write!(stream, "HTTP/1.1 {status} X\r\ncontent-type: application/json\r\n{location}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap();
|
||||
}
|
||||
None => held.push(stream),
|
||||
}
|
||||
}
|
||||
});
|
||||
(base, asked)
|
||||
}
|
||||
|
|
@ -25,23 +25,23 @@ fn git_in(dir: &Path, args: &[&str]) -> String {
|
|||
}
|
||||
|
||||
/// A working repo on disk, committed to by the test.
|
||||
struct Remote {
|
||||
pub(super) struct Remote {
|
||||
dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl Remote {
|
||||
fn new() -> Self {
|
||||
pub(super) fn new() -> Self {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
git_in(dir.path(), &["init", "--quiet", "-b", "main"]);
|
||||
Self { dir }
|
||||
}
|
||||
|
||||
fn path(&self) -> PathBuf {
|
||||
pub(super) fn path(&self) -> PathBuf {
|
||||
self.dir.path().to_path_buf()
|
||||
}
|
||||
|
||||
/// Writes files, removes paths, commits everything and returns the commit.
|
||||
fn commit(&self, write: &[(&str, &[u8])], remove: &[&str]) -> String {
|
||||
pub(super) fn commit(&self, write: &[(&str, &[u8])], remove: &[&str]) -> String {
|
||||
for (path, bytes) in write {
|
||||
let full = self.dir.path().join(path);
|
||||
std::fs::create_dir_all(full.parent().unwrap()).unwrap();
|
||||
|
|
@ -56,13 +56,13 @@ impl Remote {
|
|||
}
|
||||
}
|
||||
|
||||
struct FakeForge {
|
||||
pub(super) struct FakeForge {
|
||||
repos: Vec<(SourceRepo, PathBuf)>,
|
||||
shortfall: Option<String>,
|
||||
}
|
||||
|
||||
impl FakeForge {
|
||||
fn of(repos: &[(&str, &Remote)]) -> Self {
|
||||
pub(super) fn of(repos: &[(&str, &Remote)]) -> Self {
|
||||
Self {
|
||||
repos: repos
|
||||
.iter()
|
||||
|
|
|
|||
|
|
@ -120,10 +120,18 @@ k=60 are calibration, not design; RRF over a handful of lists is the
|
|||
established baseline and I did not survey learned fusion (it needs training
|
||||
data we lack).
|
||||
|
||||
## Status (head-maintainer pass, 2026-11-05, Huginn `25199c9`)
|
||||
## Status (2026-11-17)
|
||||
|
||||
Checked against main on merge. Nothing in this map is built: no `texts` on
|
||||
`SourceQuery`, no chat code under `sources/`. The anchors above were corrected.
|
||||
Built: cut 1 (`texts` on `SourceQuery`), chat search and window, the chat ingest
|
||||
core (`sources/chat.rs`, `room_scope.rs`, `chat_purge.rs`) and the ingest job.
|
||||
The job is `huginn-sources`: with `--matrix-url`, `--matrix-token-file`, three
|
||||
`--chat-exclude-room` ids (the estate rooms) and `--lock-file` it runs the forge
|
||||
refresh and then `refresh_chat` through `sources/matrix.rs` (GET only, bearer
|
||||
from a systemd credential file); without them it retires every chat point. The
|
||||
rooms indexed are the reader's memberships; no flag lists rooms. `huginn-sources
|
||||
chat-purge` writes a steward's purge tombstone under the same lock. What the
|
||||
operator still has to do (reader account, credential, roster, unit lines) is the
|
||||
handoff list of cut `gamecult-chat:cut_spec:cut-chat-ingest-job`.
|
||||
What changed since the map was written:
|
||||
|
||||
- Questions 1 and 2 are answered with the recommendations
|
||||
|
|
|
|||
Loading…
Reference in a new issue