From 3b8fc7da1f1c76c28315fac070068aba002e2cf3 Mon Sep 17 00:00:00 2001 From: Eureka Date: Wed, 11 Nov 2026 17:34:21 +0000 Subject: [PATCH] chat: search_chat over the shared source collection (wire, payload contract, Qdrant, queue) Probe: the deployed Qdrant answers 1.17.1 at 127.0.0.1:6333 (>= 1.8: integer range payload indexes and scroll order_by exist). Wire: SearchChat / Chat, ChatQuery (door-checked by field), ChatHit with no optional room_id, event ids or timestamps, published schemas regenerated. Index: corpus payload on every point (forge searches must corpus=forge), chat payload codec derived from the messages, keyword and integer indexes, upsert_chat / chat_nearest / chat_scroll on Qdrant and the fakes. CORPUS becomes forge:GameCult+chat:rooms, so the existing meta mismatch rebuilds the forge index once at deploy. sources::search_chat shares the identity check and fuse with the forge search; chat rides the same queue (SourceAsk) and Found::Chat. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/src/bin/huginn/main.rs | 1 + crates/eureka-state/tests/cli.rs | 47 +- crates/eureka-state/tests/common/mod.rs | 6 +- crates/huginn-daemon/src/daemon.rs | 76 +- crates/huginn-daemon/src/index/fakes.rs | 52 +- crates/huginn-daemon/src/index/mod.rs | 66 +- crates/huginn-daemon/src/index/qdrant.rs | 209 +- crates/huginn-daemon/src/index/tests.rs | 4043 +++++++++-------- crates/huginn-daemon/src/sources/mod.rs | 190 +- crates/huginn-daemon/src/sources/tests.rs | 183 +- crates/huginn-mind/src/lib.rs | 2 +- crates/huginn-mind/src/wire.rs | 144 +- .../huginn.mind_request.v1.schema.json | 69 + .../huginn.mind_response.v1.schema.json | 116 + 14 files changed, 3139 insertions(+), 2065 deletions(-) diff --git a/crates/eureka-state/src/bin/huginn/main.rs b/crates/eureka-state/src/bin/huginn/main.rs index 9f6cb10..2f7137c 100644 --- a/crates/eureka-state/src/bin/huginn/main.rs +++ b/crates/eureka-state/src/bin/huginn/main.rs @@ -688,6 +688,7 @@ fn unexpected(answer: &HuginnMindResponse) -> Trouble { HuginnMindResponse::Persona(_) => "a persona", HuginnMindResponse::PersonaPut(_) => "a persona put", HuginnMindResponse::Sources(_) => "source hits", + HuginnMindResponse::Chat(_) => "chat hits", HuginnMindResponse::Activity(_) => "repo activity", HuginnMindResponse::RootsTouched(_) => "roots touched", HuginnMindResponse::Refused(_) => "a refusal", diff --git a/crates/eureka-state/tests/cli.rs b/crates/eureka-state/tests/cli.rs index 89e652a..fafa671 100644 --- a/crates/eureka-state/tests/cli.rs +++ b/crates/eureka-state/tests/cli.rs @@ -1233,7 +1233,8 @@ struct Desk { embeds_under: &'static str, held: Vec, tickets: u64, - done: Vec<(SearchTicket, Result)>, + chat: Vec, + done: Vec<(SearchTicket, huginn_daemon::daemon::Found)>, } /// An embedder that sends every text to one vector, under a digest. @@ -1297,6 +1298,35 @@ impl huginn_daemon::sources::SourceIndex for Desk { .map(|payload| huginn_daemon::sources::SourceRow { score: 0.9, payload: payload.clone() }) .collect()) } + + fn upsert_chat(&mut self, _points: &[huginn_daemon::sources::ChatPoint]) -> anyhow::Result<()> { + anyhow::bail!("a daemon never writes the source index") + } + + fn chat_nearest( + &mut self, + _vector: &[f32], + filter: &huginn_daemon::sources::ChatFilter, + limit: u32, + _within: Duration, + ) -> anyhow::Result> { + Ok(self.chat_scroll(filter, limit, _within)?.into_iter().map(|payload| huginn_daemon::sources::ChatRow { score: 0.9, payload }).collect()) + } + + fn chat_scroll(&mut self, filter: &huginn_daemon::sources::ChatFilter, limit: u32, _within: Duration) -> anyhow::Result> { + let mut kept: Vec<_> = self + .chat + .iter() + .filter(|payload| { + let times = payload.messages.iter().map(|message| message.ts_ms); + filter.keeps(&payload.room_id, payload.room_alias.as_deref(), times.clone().min().unwrap(), times.max().unwrap()) + }) + .cloned() + .collect(); + kept.sort_by_key(|payload| payload.messages[0].ts_ms); + kept.truncate(limit as usize); + Ok(kept) + } } impl IndexSink for Desk { @@ -1313,11 +1343,21 @@ impl IndexSink for Desk { self.tickets += 1; let ticket = SearchTicket(self.tickets); let mut embedder = Flat(self.embeds_under); - let found = huginn_daemon::sources::search(&mut embedder, self, query, Duration::from_secs(5)).map_err(|error| format!("{error:#}")); + let found = huginn_daemon::sources::search(&mut embedder, self, query, Duration::from_secs(5)) + .map_or_else(|error| huginn_daemon::daemon::Found::Failed(format!("{error:#}")), huginn_daemon::daemon::Found::Sources); self.done.push((ticket, found)); Ok(ticket) } - fn sources_searched(&mut self) -> Vec<(SearchTicket, Result)> { + fn search_chat(&mut self, query: &huginn_mind::ChatQuery) -> Result { + self.tickets += 1; + let ticket = SearchTicket(self.tickets); + let mut embedder = Flat(self.embeds_under); + let found = huginn_daemon::sources::search_chat(&mut embedder, self, query, Duration::from_secs(5)) + .map_or_else(|error| huginn_daemon::daemon::Found::Failed(format!("{error:#}")), huginn_daemon::daemon::Found::Chat); + self.done.push((ticket, found)); + Ok(ticket) + } + fn sources_searched(&mut self) -> Vec<(SearchTicket, huginn_daemon::daemon::Found)> { std::mem::take(&mut self.done) } fn search_deadline(&self) -> Duration { @@ -1351,6 +1391,7 @@ fn source_daemon(built_under: &'static str, embeds_under: &'static str) -> Serve chunk("Demo", "README.md", 1, 2, "# Demo"), chunk("Other", "x.rs", 1, 1, "fn other() {}"), ], + chat: Vec::new(), tickets: 0, done: Vec::new(), }; diff --git a/crates/eureka-state/tests/common/mod.rs b/crates/eureka-state/tests/common/mod.rs index 4dd1367..075bfcf 100644 --- a/crates/eureka-state/tests/common/mod.rs +++ b/crates/eureka-state/tests/common/mod.rs @@ -93,10 +93,14 @@ impl IndexSink for Inert { fn search(&mut self, _text: &str, _top_k: u32, _among: Vec) -> Result { Err("this index cannot search".into()) } + fn search_chat(&mut self, _query: &huginn_mind::ChatQuery) -> Result { + Err("this index cannot search".into()) + } + fn search_sources(&mut self, _query: &huginn_mind::SourceQuery) -> Result { Err("this index cannot search".into()) } - fn sources_searched(&mut self) -> Vec<(SearchTicket, Result)> { + fn sources_searched(&mut self) -> Vec<(SearchTicket, huginn_daemon::daemon::Found)> { Vec::new() } fn search_deadline(&self) -> Duration { diff --git a/crates/huginn-daemon/src/daemon.rs b/crates/huginn-daemon/src/daemon.rs index 366b433..c6bad27 100644 --- a/crates/huginn-daemon/src/daemon.rs +++ b/crates/huginn-daemon/src/daemon.rs @@ -8,7 +8,7 @@ use crate::sources::forge::HttpForge; use huginn_mind::desk_wire::{DeskRequest, DeskResponse}; use huginn_mind::eureka_pipeline::{PipelineRef, Slug}; use huginn_mind::wire::{ACTIVITY_REPOS_MAX, HuginnMindRequest, HuginnMindResponse, IndexStatus}; -use huginn_mind::{Members, Mind, MindRefusal, MindStore, PipelineAdmissionOutcome, SemanticQuery, SourceHits, SourceQuery}; +use huginn_mind::{ChatHits, ChatQuery, Members, Mind, MindRefusal, MindStore, PipelineAdmissionOutcome, SemanticQuery, SourceHits, SourceQuery}; /// Where the index attaches. Called after a batch commits, with the refs the /// batch landed, derived writes included; the sink may read each through @@ -34,8 +34,12 @@ pub trait IndexSink { /// Starts a source search. The source index is not the mind's: it needs no /// member ids, and nothing about the mind's own index state refuses it. fn search_sources(&mut self, query: &SourceQuery) -> Result; - /// Every source search that finished since the last call, by ticket. - fn sources_searched(&mut self) -> Vec<(SearchTicket, Result)>; + /// Starts a chat search: the same queue and worker as a source search, + /// over the same collection. + fn search_chat(&mut self, query: &ChatQuery) -> Result; + /// Every source or chat search that finished since the last call, by + /// ticket: `Found::Sources`, `Found::Chat` or `Found::Failed`. + fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)>; fn search_deadline(&self) -> std::time::Duration; /// Every search that finished since the last call, by ticket: its /// candidates and scores, or why it failed. @@ -78,6 +82,7 @@ struct MindSearch { pub enum Found { Mind(Hits), Sources(SourceHits), + Chat(ChatHits), Failed(String), } @@ -205,6 +210,15 @@ impl> Daemon { Ok(roots) => HuginnMindResponse::RootsTouched(roots), Err(refusal) => HuginnMindResponse::Refused(refusal), }, + HuginnMindRequest::SearchChat { query, .. } => { + if let Err(refusal) = query.check() { + return Handled::Answered(HuginnMindResponse::Refused(refusal)); + } + return match self.index.search_chat(&query) { + Ok(ticket) => Handled::Searching(Search { ticket, mind: None }), + Err(detail) => Handled::Answered(HuginnMindResponse::Refused(MindRefusal::Unavailable { detail })), + }; + } HuginnMindRequest::StewardedActivity { instance, repos, after } => { let refused = |message: &str| MindRefusal::SelectionInvalid { field: "repos".into(), value: None, message: message.into() }; if repos.len() > ACTIVITY_REPOS_MAX { @@ -282,8 +296,7 @@ impl> Daemon { /// Searches the index has finished since this was last called. pub fn searched(&mut self) -> Vec<(SearchTicket, Found)> { let mind = self.index.searched().into_iter().map(|(ticket, found)| (ticket, found.map_or_else(Found::Failed, Found::Mind))); - let sources = self.index.sources_searched().into_iter().map(|(ticket, found)| (ticket, found.map_or_else(Found::Failed, Found::Sources))); - mind.chain(sources).collect() + mind.chain(self.index.sources_searched()).collect() } /// The client has stopped waiting for `ticket`. @@ -300,6 +313,7 @@ impl> Daemon { let (selection, semantic, members, hits) = match (search.mind, found) { (_, Found::Failed(detail)) => return unavailable(detail), (None, Found::Sources(hits)) => return HuginnMindResponse::Sources(hits), + (None, Found::Chat(hits)) => return HuginnMindResponse::Chat(hits), (Some(asked), Found::Mind(hits)) => (asked.selection, asked.semantic, asked.members, hits), _ => return unavailable("the index answered another kind of search than the one asked".into()), }; @@ -326,6 +340,7 @@ pub(crate) mod tests { StructuralDelta, Title, VerificationTest, }; use huginn_mind::wire::MindStatus; + use huginn_mind::ChatQuery; use huginn_mind::{MindRefusal, OwnedRedbMessagePackBackingStore}; use std::path::Path; use cultnet_rs::Selection; @@ -580,11 +595,15 @@ pub(crate) mod tests { Err("this index cannot search".into()) } + fn search_chat(&mut self, _query: &ChatQuery) -> Result { + Err("this index cannot search".into()) + } + fn search_sources(&mut self, _query: &SourceQuery) -> Result { Err("this index cannot search".into()) } - fn sources_searched(&mut self) -> Vec<(SearchTicket, Result)> { + fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> { Vec::new() } @@ -619,13 +638,17 @@ pub(crate) mod tests { Err("this index cannot search the mind".into()) } + fn search_chat(&mut self, _query: &ChatQuery) -> Result { + Err("this index cannot search".into()) + } + fn search_sources(&mut self, _query: &SourceQuery) -> Result { *self.asked.lock().unwrap() += 1; Ok(SearchTicket(7)) } - fn sources_searched(&mut self) -> Vec<(SearchTicket, Result)> { - vec![(SearchTicket(7), Ok(self.hits.clone()))] + fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> { + vec![(SearchTicket(7), Found::Sources(self.hits.clone()))] } fn search_deadline(&self) -> std::time::Duration { @@ -685,6 +708,31 @@ pub(crate) mod tests { assert_eq!(*asked.lock().unwrap(), 3, "a refused query never reached the index"); } + /// A chat search is checked at the door before the index is asked, is + /// refused for another mind, turns an index that cannot be asked into + /// `Unavailable`, and its finished hits are the answer as they are. + #[test] + fn a_chat_search_is_checked_at_the_door_and_answers_its_hits() { + use huginn_mind::eureka_pipeline::{Line, Short}; + use huginn_mind::{ChatHits, SourceModel}; + let root = tempfile::tempdir().unwrap(); + let mut daemon = Daemon::new(Mind::open(root.path(), &slug(INSTANCE)).unwrap(), NoIndex); + let request = |instance: &str, rooms: &[&str], top_k: u32| HuginnMindRequest::SearchChat { + instance: slug(instance), + query: ChatQuery { texts: vec![Line("find".into())], rooms: rooms.iter().map(|room| Short((*room).into())).collect(), after_ms: None, before_ms: None, top_k }, + }; + for (refused, field) in [(request(INSTANCE, &[], 0), "top_k"), (request(INSTANCE, &["swarm"], 5), "rooms")] { + let answer = daemon.handle(refused, now()).answered(); + assert!(matches!(answer, HuginnMindResponse::Refused(MindRefusal::SelectionInvalid { field: ref named, .. }) if named == field), "{answer:?}"); + } + assert!(matches!(daemon.handle(request(OTHER, &[], 5), now()).answered(), HuginnMindResponse::Refused(_))); + let answer = daemon.handle(request(INSTANCE, &["#swarm:x"], 5), now()).answered(); + assert!(matches!(answer, HuginnMindResponse::Refused(MindRefusal::Unavailable { .. })), "{answer:?}"); + let hits = ChatHits { indexed_under: SourceModel { name: "m".into(), digest: "d".into() }, truncated: false, hits: vec![] }; + let search = Search { ticket: SearchTicket(1), mind: None }; + assert_eq!(daemon.finish(search, Found::Chat(hits.clone())), HuginnMindResponse::Chat(hits)); + } + /// A daemon over the instance's mind under `state_root`, with no index. pub(crate) fn open_unindexed( state_root: &Path, @@ -716,11 +764,15 @@ pub(crate) mod tests { Err("this index cannot search".into()) } + fn search_chat(&mut self, _query: &ChatQuery) -> Result { + Err("this index cannot search".into()) + } + fn search_sources(&mut self, _query: &SourceQuery) -> Result { Err("this index cannot search".into()) } - fn sources_searched(&mut self) -> Vec<(SearchTicket, Result)> { + fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> { Vec::new() } @@ -750,11 +802,15 @@ pub(crate) mod tests { Err("this index cannot search".into()) } + fn search_chat(&mut self, _query: &ChatQuery) -> Result { + Err("this index cannot search".into()) + } + fn search_sources(&mut self, _query: &SourceQuery) -> Result { Err("this index cannot search".into()) } - fn sources_searched(&mut self) -> Vec<(SearchTicket, Result)> { + fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> { Vec::new() } diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs index b62ce8b..7a905fb 100644 --- a/crates/huginn-daemon/src/index/fakes.rs +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -8,7 +8,7 @@ use std::time::Duration; use anyhow::{Result, bail, ensure}; -use crate::sources::{DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta}; +use crate::sources::{ChatFilter, ChatPayload, ChatPoint, ChatRow, DescribedSources, SourceIndex, SourcePoint, SourceRow, SourcesMeta}; use super::{CollectionMeta, Described, Embedder, Hit, ModelIdentity, Point, VectorIndex, point_id}; @@ -253,6 +253,8 @@ pub(crate) fn pair() -> (FakeEmbedder, FakeIndex) { pub(crate) struct SourcesState { pub(crate) described: Option, pub(crate) points: BTreeMap, + /// The chat points, by id: the same collection, the other corpus. + pub(crate) chat: BTreeMap, /// How many times the collection was recreated. pub(crate) recreated: u32, pub(crate) upsert_calls: u32, @@ -317,6 +319,7 @@ impl SourceIndex for FakeSources { let mut state = self.state.lock().unwrap(); state.recreated += 1; state.points.clear(); + state.chat.clear(); state.described = Some(DescribedSources::Labelled { meta: meta.clone(), cursors: BTreeMap::new() }); Ok(()) } @@ -369,6 +372,44 @@ impl SourceIndex for FakeSources { rows.truncate(limit as usize); Ok(rows) } + + fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> { + let mut state = self.state.lock().unwrap(); + for point in points { + ensure!(!point.payload.messages.is_empty(), "a chat chunk carries no messages"); + state.chat.insert(point.id.clone(), point.clone()); + } + Ok(()) + } + + fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, _within: Duration) -> Result> { + let state = self.state.lock().unwrap(); + let mut rows: Vec = state + .chat + .values() + .filter(|point| chat_kept(filter, &point.payload)) + .map(|point| ChatRow { score: cosine(vector, &point.vector), payload: point.payload.clone() }) + .collect(); + rows.sort_by(|a, b| b.score.total_cmp(&a.score)); + rows.truncate(limit as usize); + Ok(rows) + } + + fn chat_scroll(&mut self, filter: &ChatFilter, limit: u32, _within: Duration) -> Result> { + let state = self.state.lock().unwrap(); + let mut kept: Vec<&ChatPayload> = state.chat.values().map(|point| &point.payload).filter(|payload| chat_kept(filter, payload)).collect(); + kept.sort_by_key(|payload| payload.messages.iter().map(|message| message.ts_ms).min()); + Ok(kept.into_iter().take(limit as usize).cloned().collect()) + } +} + +/// A chat chunk under a filter, by the span its messages make. +fn chat_kept(filter: &ChatFilter, payload: &ChatPayload) -> bool { + let times = || payload.messages.iter().map(|message| message.ts_ms); + match (times().min(), times().max()) { + (Some(start), Some(end)) => filter.keeps(&payload.room_id, payload.room_alias.as_deref(), start, end), + _ => false, + } } impl SourceIndex for FakeIndex { @@ -390,4 +431,13 @@ impl SourceIndex for FakeIndex { fn search_sources(&mut self, vector: &[f32], repo: Option<&str>, front: bool, limit: u32, within: Duration) -> Result> { self.sources.search_sources(vector, repo, front, limit, within) } + fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> { + self.sources.upsert_chat(points) + } + fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, within: Duration) -> Result> { + self.sources.chat_nearest(vector, filter, limit, within) + } + fn chat_scroll(&mut self, filter: &ChatFilter, limit: u32, within: Duration) -> Result> { + self.sources.chat_scroll(filter, limit, within) + } } diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs index 0c7c19d..167972f 100644 --- a/crates/huginn-daemon/src/index/mod.rs +++ b/crates/huginn-daemon/src/index/mod.rs @@ -105,9 +105,9 @@ use huginn_mind::wire::IndexStatus; use huginn_mind::{INDEX_TEXT_VERSION, IndexEntry, Mind, MindStore}; use sha2::{Digest, Sha256}; -use crate::daemon::{Hits, IndexSink, SearchTicket}; +use crate::daemon::{Found, Hits, IndexSink, SearchTicket}; use crate::sources::{self, SourceIndex}; -use huginn_mind::{SourceHits, SourceQuery}; +use huginn_mind::{ChatQuery, SourceQuery}; use ollama::QUERY_INSTRUCTION; /// The `managed_by` a collection carries when this organ made it. @@ -685,11 +685,17 @@ struct Queued { due: Instant, } -/// A source search waiting for the worker. It is not the mind's: nothing about -/// the mind's reconciliation gates it. +/// What a source-collection search asks: forge chunks or chat chunks. +enum SourceAsk { + Sources(SourceQuery), + Chat(ChatQuery), +} + +/// A source or chat search waiting for the worker. It is not the mind's: +/// nothing about the mind's reconciliation gates it. struct QueuedSource { ticket: SearchTicket, - query: SourceQuery, + ask: SourceAsk, due: Instant, } @@ -706,7 +712,7 @@ struct Shared { /// Searches the worker finished and the serve loop has not collected. found: Vec<(SearchTicket, Result)>, source_searches: VecDeque, - sources_found: Vec<(SearchTicket, Result)>, + sources_found: Vec<(SearchTicket, Found)>, next_ticket: u64, } @@ -821,10 +827,14 @@ fn serve_source_search(projector: &mu if guard.closed { return; } - let Some(QueuedSource { ticket, query, due }) = guard.source_searches.pop_front() else { return }; + let Some(QueuedSource { ticket, ask, due }) = guard.source_searches.pop_front() else { return }; drop(guard); - let found = sources::search(&mut projector.embedder, &mut projector.index, &query, due.saturating_duration_since(Instant::now())) - .map_err(|error| format!("{error:#}")); + let within = due.saturating_duration_since(Instant::now()); + let found = match ask { + SourceAsk::Sources(query) => sources::search(&mut projector.embedder, &mut projector.index, &query, within).map(Found::Sources), + SourceAsk::Chat(query) => sources::search_chat(&mut projector.embedder, &mut projector.index, &query, within).map(Found::Chat), + } + .unwrap_or_else(|error| Found::Failed(format!("{error:#}"))); lock(shared).sources_found.push((ticket, found)); } @@ -966,6 +976,24 @@ pub struct WorkerSink { } impl WorkerSink { + /// Queues a source or chat search behind the same bound as every other + /// search; the mind's index state does not refuse it. + fn queue_source_ask(&mut self, ask: SourceAsk) -> Result { + let mut guard = lock(&self.shared); + if guard.searches.len() + guard.source_searches.len() >= SEARCHES_QUEUED_MAX { + return Err(format!( + "the index has {} searches waiting and takes no more until it has answered some", + guard.searches.len() + guard.source_searches.len() + )); + } + let ticket = SearchTicket(guard.next_ticket); + guard.next_ticket += 1; + guard.source_searches.push_back(QueuedSource { ticket, ask, due: Instant::now() + self.deadline }); + drop(guard); + self.shared.1.notify_all(); + Ok(ticket) + } + /// Starts the worker. `mind` is the mind's genesis receipt id, `None` for /// a mind not yet written. `startup` is every indexable document the mind /// holds now; the worker's first act is to reconcile the collection @@ -1086,22 +1114,14 @@ impl IndexSink for WorkerSink { } fn search_sources(&mut self, query: &SourceQuery) -> Result { - let mut guard = lock(&self.shared); - if guard.searches.len() + guard.source_searches.len() >= SEARCHES_QUEUED_MAX { - return Err(format!( - "the index has {} searches waiting and takes no more until it has answered some", - guard.searches.len() + guard.source_searches.len() - )); - } - let ticket = SearchTicket(guard.next_ticket); - guard.next_ticket += 1; - guard.source_searches.push_back(QueuedSource { ticket, query: query.clone(), due: Instant::now() + self.deadline }); - drop(guard); - self.shared.1.notify_all(); - Ok(ticket) + self.queue_source_ask(SourceAsk::Sources(query.clone())) } - fn sources_searched(&mut self) -> Vec<(SearchTicket, Result)> { + fn search_chat(&mut self, query: &ChatQuery) -> Result { + self.queue_source_ask(SourceAsk::Chat(query.clone())) + } + + fn sources_searched(&mut self) -> Vec<(SearchTicket, Found)> { std::mem::take(&mut lock(&self.shared).sources_found) } diff --git a/crates/huginn-daemon/src/index/qdrant.rs b/crates/huginn-daemon/src/index/qdrant.rs index b897dc9..66d08e4 100644 --- a/crates/huginn-daemon/src/index/qdrant.rs +++ b/crates/huginn-daemon/src/index/qdrant.rs @@ -7,7 +7,12 @@ use std::time::Duration; use anyhow::{Context, Result, ensure}; use serde_json::{Value, json}; -use crate::sources::{COLLECTION, FRONT_PAGE_FILES, DescribedSources, SourceIndex, SourcePayload, SourcePoint, SourceRow, SourcesMeta}; +use huginn_mind::ChatMessage; + +use crate::sources::{ + COLLECTION, CORPUS_CHAT, CORPUS_FORGE, ChatFilter, ChatPayload, ChatPoint, ChatRow, DescribedSources, FRONT_PAGE_FILES, SourceIndex, SourcePayload, SourcePoint, SourceRow, + SourcesMeta, +}; use super::{CollectionMeta, INDEX_CALL_TIMEOUT, Described, Hit, Http, Method, Point, VectorIndex}; @@ -175,21 +180,94 @@ impl VectorIndex for QdrantIndex { } } -/// The payload filter of a source search: the repo, and the front-page paths -/// (exact matches on the keyword-indexed `path`), both when both are asked. -fn source_filter(repo: Option<&str>, front: bool) -> Option { - let mut must = Vec::new(); +/// The payload filter of a source search: forge points only, then the repo and +/// the front-page paths (exact matches on the keyword-indexed `path`) when +/// asked. A chat point carries no repo or path and can never pass it. +fn source_filter(repo: Option<&str>, front: bool) -> Value { + let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_FORGE } })]; if let Some(repo) = repo { must.push(json!({ "key": "repo", "match": { "value": repo } })); } if front { must.push(json!({ "key": "path", "match": { "any": FRONT_PAGE_FILES } })); } - (!must.is_empty()).then(|| json!({ "must": must })) + json!({ "must": must }) +} + +/// The payload filter of a chat read: chat points only; the rooms any-of by id +/// (`!`) or alias (`#`) as a `should`; the span as ranges that keep a chunk +/// overlapping [after, before]. +fn chat_filter(filter: &ChatFilter) -> Value { + let mut must = vec![json!({ "key": "corpus", "match": { "value": CORPUS_CHAT } })]; + if let Some(after) = filter.after_ms { + must.push(json!({ "key": "ts_end_ms", "range": { "gte": after } })); + } + if let Some(before) = filter.before_ms { + must.push(json!({ "key": "ts_start_ms", "range": { "lte": before } })); + } + let mut out = json!({ "must": must }); + let (ids, aliases): (Vec<&String>, Vec<&String>) = filter.rooms.iter().partition(|room| room.starts_with('!')); + let mut should = Vec::new(); + if !ids.is_empty() { + should.push(json!({ "key": "room_id", "match": { "any": ids } })); + } + if !aliases.is_empty() { + should.push(json!({ "key": "room_alias", "match": { "any": aliases } })); + } + if !should.is_empty() { + out["should"] = Value::Array(should); + } + out +} + +/// A chat chunk's payload. The span, first and last ids and the id list are +/// derived from `messages`, so they cannot disagree with them. +fn chat_payload_json(payload: &ChatPayload) -> Result { + let (Some(first), Some(last)) = (payload.messages.first(), payload.messages.last()) else { + anyhow::bail!("a chat chunk of {} carries no messages", payload.room_id); + }; + let mut out = json!({ + "corpus": CORPUS_CHAT, + "room_id": payload.room_id, + "ts_start_ms": payload.messages.iter().map(|message| message.ts_ms).min(), + "ts_end_ms": payload.messages.iter().map(|message| message.ts_ms).max(), + "first_event_id": first.event_id, + "last_event_id": last.event_id, + "event_ids": payload.messages.iter().map(|message| &message.event_id).collect::>(), + "messages": payload.messages.iter().map(|message| json!({ + "event_id": message.event_id, "sender": message.sender, "ts_ms": message.ts_ms, "body": message.body, + })).collect::>(), + "text": payload.text, + }); + if let Some(alias) = &payload.room_alias { + out["room_alias"] = json!(alias); + } + Ok(out) +} + +fn chat_payload_of(payload: &Value) -> Result { + ensure!(payload["corpus"] == CORPUS_CHAT, "a chat read was handed a point of another corpus"); + let text = |value: &Value, key: &str| value[key].as_str().map(str::to_owned).with_context(|| format!("a chat chunk carries no {key}")); + let messages = payload["messages"] + .as_array() + .context("a chat chunk carries no messages")? + .iter() + .map(|message| { + Ok(ChatMessage { + event_id: text(message, "event_id")?, + sender: text(message, "sender")?, + ts_ms: message["ts_ms"].as_u64().context("a chat message carries no ts_ms")?, + body: text(message, "body")?, + }) + }) + .collect::>>()?; + ensure!(!messages.is_empty(), "a chat chunk carries no messages"); + Ok(ChatPayload { room_id: text(payload, "room_id")?, room_alias: payload["room_alias"].as_str().map(str::to_owned), messages, text: text(payload, "text")? }) } fn payload_json(payload: &SourcePayload) -> Value { json!({ + "corpus": CORPUS_FORGE, "repo": payload.repo, "path": payload.path, "commit": payload.commit, @@ -270,8 +348,17 @@ impl SourceIndex for QdrantIndex { }, }); self.ask(Method::Put, &format!("{path}?wait=true"), Some(&body), &[200])?; - for field in ["repo", "path"] { - let index = json!({ "field_name": field, "field_schema": "keyword" }); + for (field, schema) in [ + ("repo", "keyword"), + ("path", "keyword"), + ("corpus", "keyword"), + ("room_id", "keyword"), + ("room_alias", "keyword"), + ("event_ids", "keyword"), + ("ts_start_ms", "integer"), + ("ts_end_ms", "integer"), + ] { + let index = json!({ "field_name": field, "field_schema": schema }); self.ask(Method::Put, &format!("{path}/index?wait=true"), Some(&index), &[200])?; } Ok(()) @@ -315,9 +402,7 @@ impl SourceIndex for QdrantIndex { let collection = self.sources.clone(); let http = self.http.bounded(within); let mut body = json!({ "vector": vector, "limit": limit, "with_payload": true, "with_vector": false }); - if let Some(filter) = source_filter(repo, front) { - body["filter"] = filter; - } + body["filter"] = source_filter(repo, front); let (_, answer) = self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?; let found = answer["result"].as_array().context("Qdrant's search has no result")?; @@ -331,6 +416,45 @@ impl SourceIndex for QdrantIndex { }) .collect() } + + fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()> { + let collection = self.sources.clone(); + let points: Vec = points + .iter() + .map(|point| Ok(json!({ "id": point.id, "vector": point.vector, "payload": chat_payload_json(&point.payload)? }))) + .collect::>()?; + self.ask(Method::Put, &format!("/collections/{collection}/points?wait=true"), Some(&json!({ "points": points })), &[200])?; + Ok(()) + } + + fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, within: Duration) -> Result> { + let collection = self.sources.clone(); + let http = self.http.bounded(within); + let body = json!({ "vector": vector, "limit": limit, "filter": chat_filter(filter), "with_payload": true, "with_vector": false }); + let (_, answer) = + self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?; + let found = answer["result"].as_array().context("Qdrant's search has no result")?; + found + .iter() + .map(|point| Ok(ChatRow { score: point["score"].as_f64().context("a hit carries no score")? as f32, payload: chat_payload_of(&point["payload"])? })) + .collect() + } + + fn chat_scroll(&mut self, filter: &ChatFilter, limit: u32, within: Duration) -> Result> { + let collection = self.sources.clone(); + let http = self.http.bounded(within); + let body = json!({ + "filter": chat_filter(filter), + "limit": limit, + "order_by": { "key": "ts_start_ms", "direction": "asc" }, + "with_payload": true, + "with_vector": false, + }); + let (_, answer) = + self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/scroll"), Some(&body), &[200])?; + let found = answer["result"]["points"].as_array().context("Qdrant's scroll has no points")?; + found.iter().map(|point| chat_payload_of(&point["payload"])).collect() + } } #[cfg(test)] @@ -339,9 +463,10 @@ mod source_filter_tests { /// No ask, no filter; a repo alone is the repo's match. #[test] - fn no_ask_is_no_filter_and_a_repo_is_its_own_match() { - assert_eq!(source_filter(None, false), None); - assert_eq!(source_filter(Some("Huginn"), false), Some(json!({ "must": [{ "key": "repo", "match": { "value": "Huginn" } }] }))); + fn no_ask_is_the_forge_corpus_and_a_repo_is_its_own_match() { + let forge = json!({ "key": "corpus", "match": { "value": "forge" } }); + assert_eq!(source_filter(None, false), json!({ "must": [forge] })); + assert_eq!(source_filter(Some("Huginn"), false), json!({ "must": [forge, { "key": "repo", "match": { "value": "Huginn" } }] })); } /// The front filter matches the path against exactly the front-page files @@ -354,15 +479,63 @@ mod source_filter_tests { assert_eq!(path.len(), 1); path[0]["match"]["any"].as_array().unwrap().iter().map(|p| p.as_str().unwrap().to_owned()).collect() }; - let front = source_filter(None, true).unwrap(); + let front = source_filter(None, true); assert_eq!(any(&front), ["README.md", "README", "readme.md", "README.markdown", "CLAUDE.md", "AGENTS.md"]); - assert_eq!(front["must"].as_array().unwrap().len(), 1, "no repo condition when none was asked"); + assert_eq!(front["must"].as_array().unwrap().len(), 2, "the corpus and the path; no repo condition when none was asked"); - let both = source_filter(Some("Zyphos"), true).unwrap(); + let both = source_filter(Some("Zyphos"), true); assert_eq!(any(&both), any(&front)); let repos: Vec<&Value> = both["must"].as_array().unwrap().iter().filter(|c| c["key"] == "repo").collect(); assert_eq!(repos.len(), 1); assert_eq!(repos[0]["match"]["value"], "Zyphos"); - assert_eq!(both["must"].as_array().unwrap().len(), 2); + assert_eq!(both["must"].as_array().unwrap().len(), 3); + } + + fn payload() -> ChatPayload { + let message = |id: &str, ts: u64| ChatMessage { event_id: id.into(), sender: "@a:gamecult.org".into(), ts_ms: ts, body: format!("body {id}") }; + ChatPayload { room_id: "!r:gamecult.org".into(), room_alias: Some("#swarm:gamecult.org".into()), messages: vec![message("$a", 10), message("$b", 20)], text: "swarm\n10 @a: body".into() } + } + + /// The chat codec is its own inverse, writes the contract's fields derived + /// from the messages, and refuses a chunk with none and a point of another corpus. + #[test] + fn the_chat_payload_round_trips_the_contract() { + let written = chat_payload_json(&payload()).unwrap(); + assert_eq!(written["corpus"], "chat"); + assert_eq!((written["ts_start_ms"].as_u64(), written["ts_end_ms"].as_u64()), (Some(10), Some(20))); + assert_eq!((written["first_event_id"].as_str(), written["last_event_id"].as_str()), (Some("$a"), Some("$b"))); + assert_eq!(written["event_ids"], json!(["$a", "$b"])); + assert_eq!(written["room_alias"], "#swarm:gamecult.org"); + assert_eq!(chat_payload_of(&written).unwrap(), payload()); + let mut no_alias = payload(); + no_alias.room_alias = None; + assert!(chat_payload_json(&no_alias).unwrap().get("room_alias").is_none()); + assert_eq!(chat_payload_of(&chat_payload_json(&no_alias).unwrap()).unwrap(), no_alias); + assert!(chat_payload_json(&ChatPayload { messages: vec![], ..payload() }).is_err()); + let forge = payload_json(&SourcePayload { repo: "r".into(), path: "p".into(), commit: "c".into(), blob: "b".into(), line_start: 1, line_end: 2, chunk_index: 0, ext: "rs".into(), text: "t".into() }); + assert_eq!(forge["corpus"], "forge"); + assert!(chat_payload_of(&forge).is_err()); + } + + /// The chat filter is the chat corpus, the rooms as an any-of `should` split + /// by id and alias, and the two ranges that keep an overlapping span. + #[test] + fn the_chat_filter_is_corpus_rooms_and_overlap() { + let filter = chat_filter(&ChatFilter { rooms: vec!["!r:x".into(), "#a:x".into(), "#b:x".into()], after_ms: Some(5), before_ms: Some(9) }); + assert_eq!( + filter, + json!({ + "must": [ + { "key": "corpus", "match": { "value": "chat" } }, + { "key": "ts_end_ms", "range": { "gte": 5 } }, + { "key": "ts_start_ms", "range": { "lte": 9 } }, + ], + "should": [ + { "key": "room_id", "match": { "any": ["!r:x"] } }, + { "key": "room_alias", "match": { "any": ["#a:x", "#b:x"] } }, + ], + }) + ); + assert_eq!(chat_filter(&ChatFilter::default()), json!({ "must": [{ "key": "corpus", "match": { "value": "chat" } }] })); } } diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index dfd16af..e6e04d0 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -1,1129 +1,1129 @@ -//! The projection's rules, against in-memory ports; the adapters' request and -//! answer shapes, against a stub that speaks just enough HTTP. No test needs a -//! live Qdrant or Ollama. - -use std::collections::BTreeSet; -use std::io::{Read, Write}; -use std::net::{TcpListener, TcpStream}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, IndexStatus}; -use huginn_mind::{Mind, OwnedRedbMessagePackBackingStore, PipelineAdmissionOutcome}; -use tempfile::TempDir; - -use super::fakes::{FakeEmbedder, FakeIndex, Gate, Stored, pair}; -use super::ollama::OllamaEmbedder; -use super::qdrant::QdrantIndex; -use super::*; -use crate::daemon::Daemon; -use crate::daemon::tests::{INSTANCE, OTHER, batch, campaign_seed, now, question_and_ruling, slug}; - -type AfterAbsorb = Box; -type Edit = (&'static str, Box); - -thread_local! { - /// Runs once, in the worker's own turn, between the inbox being absorbed - /// and a search being taken, holding the guard `serve_search` holds. - static AFTER_ABSORB: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; -} - -/// The hook `serve_search` calls at the point where a second lock once left a -/// gap. It is the test's only way in: nothing else can reach that point. -pub(super) fn after_absorb(guard: &mut Shared) { - if let Some(hook) = AFTER_ABSORB.with(|hook| hook.borrow_mut().take()) { - hook(guard); - } -} - -fn collection() -> String { - collection_name(&slug(INSTANCE)) -} - -/// The bound the adapter tests give a call. -const TIME: Duration = Duration::from_secs(30); - -/// A mind identity for tests that project entries without a mind behind them. -const MIND: &str = "mind-commit-first"; - -/// A mind of the same instance name whose first batch differs, so its genesis -/// receipt id does too. -fn other_mind_of_the_same_name() -> (TempDir, Mind) { - let root = tempfile::tempdir().unwrap(); - let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - let (question, _) = question_and_ruling(); - let mut first = campaign_seed(); - first.push(question); - let outcome = mind.admit(batch(INSTANCE, first), now()); - assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); - (root, mind) -} - -/// A real mind holding a campaign, a question, the ruling that answers it and -/// the resolution admission derived: four indexable documents, beside the -/// identity and the stewardship, which carry no text. -fn mind_with_documents() -> (TempDir, Mind) { - let root = tempfile::tempdir().unwrap(); - let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - let (question, ruling) = question_and_ruling(); - for documents in [campaign_seed(), vec![question, ruling]] { - let outcome = mind.admit(batch(INSTANCE, documents), now()); - assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); - } - (root, mind) -} - -/// How many times the collection was created and how many upserts it took. -fn counts(index: &FakeIndex) -> (usize, usize) { - let state = index.state.lock().unwrap(); - (state.recreated.len(), state.upserts.len()) -} - -/// Steps until the projector has nothing to do. A step that never settles is a -/// failure of the test, not a hang: it is bounded. -fn drain(projector: &mut Projector) { - for _ in 0..1000 { - if projector.advance().unwrap() == Advance::Idle { - return; - } - } - panic!("the projector was still stepping after 1000 steps"); -} - -fn point_ids(entries: &[IndexEntry]) -> BTreeSet { - entries.iter().map(|entry| point_id(&entry.id.id.0)).collect() -} - -fn projected(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - drain(&mut projector); - projector -} - -/// A projector whose collection was lost while it ran and has been made again: -/// it owes every entry, and is rebuilding. -fn rebuilding(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { - let mut projector = projected(embedder, index, &entries[..1]); - index.state.lock().unwrap().collections.remove(&collection()); - projector.want(entries[1..].iter().cloned()); - projector.advance().expect_err("the write meets no collection"); - assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the reconciliation makes the collection again"); - projector -} - -/// P4's function, as voidbot's `toQdrantPointId` computes it: the first sixteen -/// bytes of the SHA-256, version and variant bits set. The fixed vectors were -/// computed independently of this crate. -#[test] -fn point_ids_are_a_pure_function_of_the_document_id() { - assert_eq!(point_id("eureka-state:question:Q1"), "1e8a661b-4b7e-5245-a9ad-b15a3fa27031"); - assert_eq!(point_id("eureka-state:campaign:self"), "d4135998-051d-57fb-9360-82964874fbc9"); - assert_eq!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q1")); - assert_ne!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q2")); -} - -/// The projection converges by set difference: after a crash that lost two -/// points, a restart embeds and writes those two, and nothing else, and the -/// collection is not rebuilt. -#[test] -fn reconcile_indexes_exactly_the_indexable_documents_the_collection_lacks() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - assert_eq!(entries.len(), 4); - - let (embedder, index) = pair(); - projected(&embedder, &index, &entries); - assert_eq!( - index.holding(&collection()), - point_ids(&entries), - "one point per indexable document, none for the identity or the stewardship" - ); - - let lost: Vec = entries.iter().take(2).cloned().collect(); - { - let mut state = index.state.lock().unwrap(); - let stored = state.collections.get_mut(&collection()).unwrap(); - for entry in &lost { - stored.points.remove(&point_id(&entry.id.id.0)); - } - state.upserts.clear(); - state.recreated.clear(); - } - - let restarted = FakeEmbedder::new("d1"); - projected(&restarted, &index, &entries); - let state = index.state.lock().unwrap(); - assert!(state.recreated.is_empty(), "a collection that agrees is not rebuilt"); - let written: BTreeSet = state.upserts.concat().into_iter().collect(); - assert_eq!(written, point_ids(&lost), "exactly the missing points were written"); - assert_eq!(state.upserts.concat().len(), 2, "each once"); - assert_eq!(restarted.texts().len(), 2, "and only their text was embedded"); - drop(state); - assert_eq!(index.holding(&collection()), point_ids(&entries)); -} - -/// Compatibility is the model by name and digest, its dimensions and the text -/// version. Any of them differing rebuilds the collection whole, and no vector -/// of one model survives beside another's. -#[test] -fn a_model_digest_change_rebuilds_the_collection() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let changes: [Edit; 3] = [ - ("digest", Box::new(|identity| identity.digest = "d2".into())), - ("name", Box::new(|identity| identity.name = "other-model".into())), - ("dimensions", Box::new(|identity| identity.dimensions = 8)), - ]; - for (what, change) in changes { - let (embedder, index) = pair(); - projected(&embedder, &index, &entries); - assert_eq!(index.state.lock().unwrap().recreated.len(), 1, "the first run creates the collection"); - - let second = FakeEmbedder::new("d1"); - change(&mut second.state.lock().unwrap().identity); - let expected = second.state.lock().unwrap().identity.clone(); - projected(&second, &index, &entries); - let state = index.state.lock().unwrap(); - assert_eq!(state.recreated.len(), 2, "a changed {what} rebuilds"); - let (_, meta) = state.recreated.last().unwrap(); - assert_eq!( - (&meta.model, &meta.model_digest, meta.dimensions), - (&expected.name, &expected.digest, expected.dimensions), - "{what}" - ); - assert_eq!(second.texts().len(), 4, "every document is embedded again under {what}"); - assert_eq!(state.collections[&collection()].points.len(), 4); - } - - // The same model, digest and dimensions: nothing is rebuilt or rewritten. - let (embedder, index) = pair(); - projected(&embedder, &index, &entries); - let same = FakeEmbedder::new("d1"); - projected(&same, &index, &entries); - assert_eq!(index.state.lock().unwrap().recreated.len(), 1); - assert!(same.texts().is_empty()); - - // A text version the collection was not built with is a rebuild too. - { - let mut state = index.state.lock().unwrap(); - let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; - meta.index_text_version += 1; - } - projected(&FakeEmbedder::new("d1"), &index, &entries); - assert_eq!(index.state.lock().unwrap().recreated.len(), 2); -} - -/// No cross-mind writes: a collection that another writer or another instance -/// made, or that carries no label, is left exactly as it was. -#[test] -fn a_collection_owned_by_another_instance_is_never_recreated() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let ours = |managed_by: &str, instance: &str, mind: &str| CollectionMeta { - managed_by: managed_by.into(), - instance: instance.into(), - mind: mind.into(), - model: "model".into(), - model_digest: "d1".into(), - dimensions: 4, - index_text_version: INDEX_TEXT_VERSION, - }; - for foreign in [ - Described::Unlabelled, - Described::Labelled(ours(MANAGED_BY, OTHER, MIND)), - Described::Labelled(ours("voidbot", INSTANCE, MIND)), - Described::Labelled(ours(MANAGED_BY, INSTANCE, "mind-commit-another")), - Described::Labelled(ours(MANAGED_BY, INSTANCE, "")), - ] { - let (embedder, index) = pair(); - index.plant(&collection(), foreign.clone()); - let sentinel = point_id("someone-elses-document"); - index.state.lock().unwrap().collections.get_mut(&collection()).unwrap().points.insert( - sentinel.clone(), - Point { - id: sentinel.clone(), - vector: vec![0.0; 4], - payload: PointPayload { - doc_id: "someone-elses-document".into(), - kind: "x".into(), - root: "x".into(), - ordinal: 1, - text_sha256: "x".into(), - }, - }, - ); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - let error = projector.reconcile().expect_err("a foreign collection is refused"); - assert!(matches!(error, StepError::Refused(_)), "{foreign:?}: {error}"); - assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Refused { pending: 4, .. })); - let state = index.state.lock().unwrap(); - assert!(state.recreated.is_empty() && state.upserts.is_empty(), "{foreign:?} was written to"); - assert_eq!(state.collections[&collection()].label, foreign); - assert!(state.collections[&collection()].points.contains_key(&sentinel), "{foreign:?} lost a point"); - assert!(embedder.texts().is_empty()); - } -} - -/// The vector's length is checked against the collection's, so a flush that -/// the model did not change is still refused when the embedder answers wrongly. -#[test] -fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - projector.reconcile().unwrap(); - embedder.state.lock().unwrap().vector_length = Some(8); - let error = projector.flush_batch().expect_err("a longer vector is refused"); - assert!(matches!(error, StepError::Unavailable(_)), "{error}"); - assert_eq!(projector.pending(), 4); - assert!(index.state.lock().unwrap().upserts.is_empty()); -} - -/// A flush embeds through the same check as a search: a model that changed -/// before the embed is never asked, a model that changed during it has its -/// vectors dropped, and a collection that is no longer this mind's is refused. -/// In each case nothing is written, the documents stay wanted, and the -/// reconciliation is forgotten. -#[test] -fn a_flush_is_refused_as_a_search_is_when_the_model_or_the_collection_changed() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let reconciled = || { - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - projector.reconcile().unwrap(); - (embedder, index, projector) - }; - - let (embedder, index, mut projector) = reconciled(); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - projector.flush_batch().expect("a changed model is not a failure"); - assert!(embedder.texts().is_empty(), "before: the texts were not embedded under the new model"); - assert!(index.state.lock().unwrap().upserts.is_empty()); - assert_eq!((projector.pending(), projector.stale()), (4, true), "before"); - - let (embedder, index, mut projector) = reconciled(); - change_label(&index, |meta| meta.mind = "mind-commit-another".into()); - let error = projector.flush_batch().expect_err("another mind's label is refused"); - assert!(error.to_string().contains("no longer the one this mind's index was built as"), "{error}"); - assert!(embedder.texts().is_empty(), "label: nothing was embedded"); - assert!(index.state.lock().unwrap().upserts.is_empty()); - assert_eq!((projector.pending(), projector.stale()), (4, true), "label"); - - let gate = Gate::shut(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let index = FakeIndex::default(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - projector.reconcile().unwrap(); - let flushing = std::thread::spawn(move || { - let flushed = projector.flush_batch(); - (projector, flushed) - }); - gate.wait_arrived(1); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - gate.open(); - let (projector, flushed) = flushing.join().unwrap(); - flushed.expect("a changed model is not a failure"); - assert!(index.state.lock().unwrap().upserts.is_empty(), "during: the vectors were not written"); - assert_eq!((projector.pending(), projector.stale()), (4, true), "during"); -} - -fn change_label(index: &FakeIndex, change: impl FnOnce(&mut CollectionMeta)) { - let mut state = index.state.lock().unwrap(); - let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; - change(meta); -} -/// Every call on the search path is bounded by what remains of the search's own -/// time: the identity reads, the label read, the embed and the query. A search -/// whose embed cannot be recalled therefore cannot hold the writes off past the -/// moment nobody is waiting for it. A bulk flush keeps its own, longer, bound. -#[test] -fn every_call_on_the_search_path_is_bounded_by_the_searchs_time_and_the_flush_by_its_own() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - { - let state = embedder.state.lock().unwrap(); - assert!(!state.bounds.is_empty()); - let near = |bound: &Duration| *bound <= BULK_EMBED_TIMEOUT && *bound > BULK_EMBED_TIMEOUT - Duration::from_secs(60); - assert!(state.bounds.iter().all(near), "the flushes' embeds: {:?}", state.bounds); - assert!(state.identity_bounds.iter().all(near), "the reconciliation's and the flushes' identity reads: {:?}", state.identity_bounds); - } - embedder.state.lock().unwrap().bounds.clear(); - embedder.state.lock().unwrap().identity_bounds.clear(); - index.state.lock().unwrap().bounds.clear(); - - let within = Duration::from_secs(20); - projector.search("anything", 3, &[], within).unwrap(); - let embedder = embedder.state.lock().unwrap(); - let index = index.state.lock().unwrap(); - assert_eq!(embedder.identity_bounds.len(), 2, "the model is read before and after the embed"); - assert_eq!(embedder.bounds.len(), 1); - assert_eq!(index.bounds.iter().map(|(call, _)| *call).collect::>(), ["describe", "search"]); - let bounds: Vec = embedder - .identity_bounds - .iter() - .chain(&embedder.bounds) - .copied() - .chain(index.bounds.iter().map(|(_, bound)| *bound)) - .collect(); - assert!(bounds.iter().all(|bound| !bound.is_zero() && *bound <= within), "{bounds:?}"); - assert_eq!(Backoff::default().search_deadline, SEARCH_DEADLINE); - assert!(SEARCH_DEADLINE < BULK_EMBED_TIMEOUT); -} - -/// A hung Ollama does not hold the worker past the search that asked: the -/// identity read never answers, and the search is answered, and the worker -/// free, when the search's own time has passed. A search that waited past its -/// time in the queue is answered without asking the embedder anything. A -/// ticket's time is stamped by the sink (see `the_sink_owns_the_searchs_deadline_and_tells_the_loop`). -#[test] -fn a_hung_embedder_holds_the_worker_no_longer_than_the_search_that_asked() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - - ask_sink(&mut sink, "the ticket's time", 3).unwrap(); - answers(&mut sink, 1); - - embedder.state.lock().unwrap().hang_identity = true; - let started = Instant::now(); - let due = started + Duration::from_millis(400); - lock(&sink.shared).searches.push_back(Queued { ticket: SearchTicket(70), text: "hung".into(), top_k: 3, among: vec![], due }); - sink.shared.1.notify_all(); - let answered = answers(&mut sink, 1); - assert_eq!(answered[0].0, SearchTicket(70)); - assert!(answered[0].1.is_err(), "{answered:?}"); - assert!(started.elapsed() < Duration::from_secs(5), "the worker was held for {:?}", started.elapsed()); - - embedder.state.lock().unwrap().hang_identity = false; - let reads = embedder.state.lock().unwrap().identity_bounds.len(); - lock(&sink.shared).searches.push_back(Queued { - ticket: SearchTicket(71), - text: "late".into(), - top_k: 3, - among: vec![], - due: Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now), - }); - sink.shared.1.notify_all(); - let late = answers(&mut sink, 1); - let Err(why) = &late[0].1 else { panic!("{late:?}") }; - assert!(why.contains("has run out"), "{why}"); - assert_eq!(embedder.state.lock().unwrap().identity_bounds.len(), reads, "nothing was asked for a search nobody waits for"); - - let ticket = ask_sink(&mut sink, "the worker is free", 3).expect("the worker is still current"); - assert_eq!(answers(&mut sink, 1)[0].0, ticket); -} - -/// The Ollama adapter enforces the bound it is given: an embed that is never -/// answered fails at the bound, not at the adapter's default. -#[test] -fn an_ollama_embed_that_is_never_answered_fails_at_its_bound() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - let held = std::thread::spawn(move || { - let mut held = Vec::new(); - for stream in listener.incoming().take(1) { - held.push(stream.unwrap()); - std::thread::sleep(Duration::from_secs(8)); - } - }); - let (sent, received) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let mut embedder = OllamaEmbedder::new(&url, "m:1"); - sent.send(embedder.embed(&["a".into()], Duration::from_millis(300)).is_err()).ok(); - }); - assert_eq!(received.recv_timeout(Duration::from_secs(5)), Ok(true), "the embed did not fail at its bound"); - drop(held); -} -fn whoami_index(daemon: &mut Daemon) -> IndexStatus { - match daemon.handle(HuginnMindRequest::Whoami, now()).answered() { - HuginnMindResponse::Whoami(status) => status.index, - other => panic!("expected a status, got {other:?}"), - } -} - -fn committed(response: HuginnMindResponse) { - assert!( - matches!(response, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), - "{response:?}" - ); -} - -fn quick() -> Backoff { - Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) } -} - -/// The index never delays or refuses admission: the embedder is shut, the -/// worker is inside it, and two more admissions are answered at once. The gate -/// counts arrivals, so this observes where the work is and not how long it -/// took. -#[test] -fn an_admission_replies_while_the_embedder_is_blocked() { - let root = tempfile::tempdir().unwrap(); - let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - let gate = Gate::shut(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let index = FakeIndex::default(); - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), mind.index_entries(None).unwrap(), quick()); - let mut daemon = Daemon::new(mind, sink); - daemon.index().wait_status(|status| *status == IndexStatus::Current); - - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, campaign_seed())), now()).answered()); - gate.wait_arrived(1); - let (question, ruling) = question_and_ruling(); - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()).answered()); - assert_eq!( - whoami_index(&mut daemon), - IndexStatus::Behind { pending: 4 }, - "the campaign is in the embedder and three more are queued" - ); - assert!(index.holding(&collection()).is_empty(), "nothing has been written yet"); - - gate.open(); - daemon.index().wait_status(|status| *status == IndexStatus::Current); - assert_eq!(index.holding(&collection()).len(), 4); - assert_eq!(whoami_index(&mut daemon), IndexStatus::Current); -} - -/// A failing embedder is reported by `whoami`, retried on the backoff, and -/// nothing is dropped: when it returns, every document is written. -#[test] -fn a_failing_embedder_is_visible_in_whoami_and_retried_never_dropped() { - let root = tempfile::tempdir().unwrap(); - let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); - let mut daemon = Daemon::new(mind, sink); - daemon.index().wait_status(|status| *status == IndexStatus::Current); - - embedder.state.lock().unwrap().down = true; - let (question, ruling) = question_and_ruling(); - for documents in [campaign_seed(), vec![question, ruling]] { - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), now()).answered()); - } - let seen = daemon.index().wait_status(|status| matches!(status, IndexStatus::Failing { attempts, .. } if *attempts >= 2)); - let IndexStatus::Failing { pending, error, .. } = seen else { panic!() }; - assert_eq!(pending, 4); - assert!(error.contains("the embedder is down"), "{error}"); - assert!(matches!(whoami_index(&mut daemon), IndexStatus::Failing { .. })); - assert!(index.holding(&collection()).is_empty()); - - embedder.state.lock().unwrap().down = false; - daemon.index().wait_status(|status| *status == IndexStatus::Current); - assert_eq!(index.holding(&collection()).len(), 4, "every admitted document was kept and written"); -} - -/// A collection that is not this mind's is reported as refused, never -/// recreated, and re-read on the backoff: once it is gone the projection -/// completes. -#[test] -fn a_refused_collection_is_reported_and_reread() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - index.plant(&collection(), Described::Unlabelled); - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries, quick()); - let mut daemon = Daemon::new(mind, sink); - daemon.index().wait_status(|status| matches!(status, IndexStatus::Refused { attempts, .. } if *attempts >= 2)); - assert!(matches!(whoami_index(&mut daemon), IndexStatus::Refused { pending: 4, .. })); - assert!(index.state.lock().unwrap().recreated.is_empty()); - - index.state.lock().unwrap().collections.remove(&collection()); - daemon.index().wait_status(|status| *status == IndexStatus::Current); - assert_eq!(index.holding(&collection()).len(), 4); -} - -// The adapters, against a stub that answers the routes it is given. - -struct Stub { - url: String, - seen: Arc>>, -} - -type Route = (&'static str, u16, &'static str); - -fn find(haystack: &[u8], needle: &[u8]) -> Option { - haystack.windows(needle.len()).position(|window| window == needle) -} - -/// Serves each route once, in order, then repeats the last route that matched. -/// A route matches `" "` by prefix. -fn stub(routes: &[Route]) -> Stub { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - let seen = Arc::new(Mutex::new(Vec::new())); - let routes: Arc>> = Arc::new(Mutex::new(routes.iter().map(|route| (*route, false)).collect())); - let log = Arc::clone(&seen); - std::thread::spawn(move || { - for stream in listener.incoming() { - let Ok(stream) = stream else { break }; - let (routes, log) = (Arc::clone(&routes), Arc::clone(&log)); - std::thread::spawn(move || answer_connection(stream, &routes, &log)); - } - }); - Stub { url, seen } -} - -fn answer_connection( - mut stream: TcpStream, - routes: &Mutex>, - log: &Mutex>, -) { - let mut buffer: Vec = Vec::new(); - loop { - let header_end = loop { - if let Some(at) = find(&buffer, b"\r\n\r\n") { - break at + 4; - } - let mut chunk = [0_u8; 4096]; - match stream.read(&mut chunk) { - Ok(0) | Err(_) => return, - Ok(read) => buffer.extend_from_slice(&chunk[..read]), - } - }; - let head = String::from_utf8_lossy(&buffer[..header_end]).into_owned(); - let length = head - .lines() - .find_map(|line| line.to_ascii_lowercase().strip_prefix("content-length:").map(|value| value.trim().parse::().unwrap())) - .unwrap_or(0); - while buffer.len() < header_end + length { - let mut chunk = [0_u8; 4096]; - match stream.read(&mut chunk) { - Ok(0) | Err(_) => return, - Ok(read) => buffer.extend_from_slice(&chunk[..read]), - } - } - let body = String::from_utf8_lossy(&buffer[header_end..header_end + length]).into_owned(); - buffer.drain(..header_end + length); - let mut first = head.lines().next().unwrap().split(' '); - let (method, path) = (first.next().unwrap().to_owned(), first.next().unwrap().to_owned()); - log.lock().unwrap().push((method.clone(), path.clone(), body)); - let key = format!("{method} {path}"); - let (status, answer) = { - let mut routes = routes.lock().unwrap(); - let matching: Vec = - routes.iter().enumerate().filter(|(_, ((prefix, ..), _))| key.starts_with(prefix)).map(|(at, _)| at).collect(); - let chosen = matching.iter().copied().find(|at| !routes[*at].1).or(matching.last().copied()); - match chosen { - Some(at) => { - routes[at].1 = true; - (routes[at].0.1, routes[at].0.2) - } - None => (404, "{}"), - } - }; - let response = - format!("HTTP/1.1 {status} X\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{answer}", answer.len()); - if stream.write_all(response.as_bytes()).is_err() { - return; - } - } -} - -const TAGS: &str = r#"{"models":[{"name":"other:1","model":"other:1","digest":"zzz"},{"name":"m:1","model":"m:1","digest":"ac6d"}]}"#; -const SHOW: &str = r#"{"model_info":{"general.architecture":"qwen3","qwen3.embedding_length":4}}"#; -const EMBED: &str = r#"{"embeddings":[[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0]]}"#; - -#[test] -fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { - let ollama = stub(&[("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]); - let mut embedder = OllamaEmbedder::new(&ollama.url, "m:1"); - assert_eq!(embedder.model_identity(TIME).unwrap(), ModelIdentity { name: "m:1".into(), digest: "ac6d".into(), dimensions: 4 }); - let vectors = embedder.embed(&["a".into(), "b".into(), "c".into(), "d".into()], Duration::from_secs(5)).unwrap(); - assert_eq!(vectors.len(), 4); - assert_eq!(vectors[0], vec![0.5, 1.0, 2.0, 3.0]); - let seen = ollama.seen.lock().unwrap(); - let (_, _, body) = seen.iter().find(|(_, path, _)| path == "/api/embed").unwrap(); - assert!(body.contains("\"model\":\"m:1\"") && body.contains("\"input\":[\"a\",\"b\",\"c\",\"d\"]"), "{body}"); - drop(seen); - - // A listing entry that carries only `name` is still the model. - let bare = stub(&[ - ("GET /api/tags", 200, r#"{"models":[{"name":"m:1","digest":"ac6d"}]}"#), - ("POST /api/show", 200, SHOW), - ]); - assert_eq!(OllamaEmbedder::new(&bare.url, "m:1").model_identity(TIME).unwrap().digest, "ac6d"); - - let mut absent = OllamaEmbedder::new(&ollama.url, "not-pulled:1"); - assert!(format!("{:#}", absent.model_identity(TIME).unwrap_err()).contains("does not list the model")); - let refusing = stub(&[("POST /api/embed", 500, r#"{"error":"boom"}"#)]); - assert!(OllamaEmbedder::new(&refusing.url, "m:1").embed(&["a".into()], Duration::from_secs(5)).is_err()); -} - -/// Compatibility never reads the address: the same model at another address -/// leaves the collection alone, and the same address serving another digest -/// rebuilds it. -#[test] -fn the_embedders_address_is_not_part_of_the_collections_compatibility() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let routes = [("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]; - let repulled = TAGS.replace("ac6d", "beef"); - let index = FakeIndex::default(); - - let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); - first.want(entries.iter().cloned()); - drain(&mut first); - assert_eq!(counts(&index), (1, 1)); - - // Another address, the same model and digest. - let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); - moved.want(entries.iter().cloned()); - drain(&mut moved); - assert_eq!(counts(&index), (1, 1)); - - // The same model name re-pulled: another digest. - let leaked: &'static str = Box::leak(repulled.into_boxed_str()); - let other = [("GET /api/tags", 200, leaked), routes[1], routes[2]]; - let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); - rebuilt.want(entries.iter().cloned()); - drain(&mut rebuilt); - assert_eq!(counts(&index), (2, 2)); -} - -#[test] -fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { - let name = collection(); - let absent = stub(&[("GET /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); - assert_eq!(QdrantIndex::new(&absent.url).describe(&name, TIME).unwrap(), Described::Absent); - - let unlabelled = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":null}}}"#)]); - assert_eq!(QdrantIndex::new(&unlabelled.url).describe(&name, TIME).unwrap(), Described::Unlabelled); - let voidbot = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":{"managedBy":"voidbot"}}}}"#)]); - assert_eq!(QdrantIndex::new(&voidbot.url).describe(&name, TIME).unwrap(), Described::Unlabelled); - - let labelled = stub(&[( - "GET /collections/", - 200, - r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","mind":"mind-commit-first","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, - )]); - assert_eq!( - QdrantIndex::new(&labelled.url).describe(&name, TIME).unwrap(), - Described::Labelled(CollectionMeta { - managed_by: "huginn".into(), - instance: "yggdrasil".into(), - mind: "mind-commit-first".into(), - model: "m:1".into(), - model_digest: "ac6d".into(), - dimensions: 4, - index_text_version: 1, - }) - ); - - // A scroll is followed to its last page. - let paged = stub(&[ - ("POST /collections/", 200, r#"{"result":{"points":[{"id":"a"},{"id":"b"}],"next_page_offset":"b"}}"#), - ("POST /collections/", 200, r#"{"result":{"points":[{"id":"c"}],"next_page_offset":null}}"#), - ]); - let ids = QdrantIndex::new(&paged.url).ids(&name).unwrap(); - let expected: BTreeSet = ["a", "b", "c"].map(String::from).into_iter().collect(); - assert_eq!(ids, expected); - let seen = paged.seen.lock().unwrap(); - assert!(!seen[0].2.contains("offset") && seen[1].2.contains("\"offset\":\"b\""), "{seen:?}"); - drop(seen); - - // A rebuild drops, then creates with the metadata and the vector shape. - let writing = stub(&[("DELETE /collections/", 200, "{}"), ("PUT /collections/", 200, "{}")]); - let meta = CollectionMeta { - managed_by: MANAGED_BY.into(), - instance: INSTANCE.into(), - mind: MIND.into(), - model: "m:1".into(), - model_digest: "ac6d".into(), - dimensions: 4, - index_text_version: INDEX_TEXT_VERSION, - }; - let mut store = QdrantIndex::new(&writing.url); - store.recreate(&name, &meta).unwrap(); - let point = Point { - id: point_id("d"), - vector: vec![1.0, 2.0, 3.0, 4.0], - payload: PointPayload { doc_id: "d".into(), kind: "question".into(), root: "eureka-state".into(), ordinal: 7, text_sha256: "h".into() }, - }; - store.upsert(&name, &[point]).unwrap(); - let seen = writing.seen.lock().unwrap(); - let paths: Vec = seen.iter().map(|(method, path, _)| format!("{method} {path}")).collect(); - assert_eq!( - paths, - [ - format!("DELETE /collections/{name}?wait=true"), - format!("PUT /collections/{name}?wait=true"), - format!("PUT /collections/{name}/points?wait=true"), - ] - ); - for expected in ["\"size\":4", "\"distance\":\"Cosine\"", "\"on_disk\":true", "\"managed_by\":\"huginn\"", "\"index_text_version\":1"] { - assert!(seen[1].2.contains(expected), "{expected} in {}", seen[1].2); - } - for expected in ["\"doc_id\":\"d\"", "\"kind\":\"question\"", "\"root\":\"eureka-state\"", "\"ordinal\":7", "\"text_sha256\":\"h\""] { - assert!(seen[2].2.contains(expected), "{expected} in {}", seen[2].2); - } - - let failing = stub(&[("PUT /collections/", 500, r#"{"status":{"error":"disk full"}}"#)]); - assert!(QdrantIndex::new(&failing.url).upsert(&name, &[]).is_err()); -} - -/// The collection's name is the contract with the store: one per instance, -/// `huginn_mind_`. -#[test] -fn the_collection_is_named_for_the_instance() { - assert_eq!(collection_name(&slug("yggdrasil")), "huginn_mind_yggdrasil"); - assert_eq!(collection_name(&slug("a.b")), "huginn_mind_a.b"); -} - -#[test] -fn a_step_error_says_what_stopped_it() { - assert_eq!(StepError::Refused("not ours".into()).to_string(), "not ours"); - let unavailable = StepError::Unavailable(anyhow::anyhow!("inner").context("outer")); - assert_eq!(unavailable.to_string(), "outer: inner"); -} - -/// What the sink reports is the worker's health with the inbox added to the -/// pending count, and a current index with something queued is behind. -#[test] -fn queued_entries_count_toward_every_status_that_has_a_pending_count() { - use IndexStatus as S; - assert_eq!(including(&S::Current, 0), S::Current); - assert_eq!(including(&S::Current, 3), S::Behind { pending: 3 }); - assert_eq!(including(&S::Reconciling { pending: 2 }, 3), S::Reconciling { pending: 5 }); - assert_eq!(including(&S::Behind { pending: 2 }, 3), S::Behind { pending: 5 }); - assert_eq!( - including(&S::Failing { pending: 2, attempts: 4, error: "e".into() }, 3), - S::Failing { pending: 5, attempts: 4, error: "e".into() } - ); - assert_eq!( - including(&S::Refused { pending: 2, attempts: 4, reason: "r".into() }, 3), - S::Refused { pending: 5, attempts: 4, reason: "r".into() } - ); -} - -/// Dropping the sink stops the worker: the worker's hold on the shared state -/// is released. A worker that never stopped would hold it forever. -#[test] -fn dropping_the_sink_stops_the_worker() { - let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let shared = Arc::downgrade(&sink.shared); - drop(sink); - eventually("the worker letting go of the shared state", || shared.upgrade().is_none()); -} - -/// F1. The promised curve, pure: 30 s doubling to a 10 min cap, never below -/// the first delay and never past the cap. A retry that shrank would hot-loop -/// against a dead Ollama. -#[test] -fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() { - let backoff = Backoff::default(); - assert_eq!((backoff.initial, backoff.max), (Duration::from_secs(30), Duration::from_secs(600))); - assert_eq!(backoff.recheck, Duration::from_secs(60), "an idle worker verifies the model and the collection each minute"); - let mut delay = backoff.initial; - let mut seen = vec![delay.as_secs()]; - for _ in 0..7 { - let next = backoff.after(delay); - assert!(next >= delay && next >= backoff.initial, "the delay never shrinks: {delay:?} -> {next:?}"); - assert!(next <= backoff.max, "the delay never passes the cap: {next:?}"); - delay = next; - seen.push(delay.as_secs()); - } - assert_eq!(seen, [30, 60, 120, 240, 480, 600, 600, 600]); - let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) }; - assert_eq!(small.after(Duration::from_millis(1)), Duration::from_millis(2)); - assert_eq!(small.after(Duration::from_millis(2)), Duration::from_millis(3)); -} - -/// F2. A collection lost while the daemon runs is found by the next failed -/// write, recreated and refilled from what the projector knows, without a -/// restart. -#[test] -fn a_collection_lost_after_startup_is_recreated_and_refilled() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - assert_eq!(projector.progress_status(), IndexStatus::Current); - assert_eq!(index.holding(&collection()).len(), 3); - - index.state.lock().unwrap().collections.remove(&collection()); - projector.want(entries[3..].iter().cloned()); - let error = projector.advance().expect_err("the write meets no collection"); - assert!(matches!(error, StepError::Unavailable(_)), "{error}"); - assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Failing { pending: 1, .. })); - - drain(&mut projector); - assert_eq!(index.state.lock().unwrap().recreated.len(), 2, "the collection was made again"); - assert_eq!(index.holding(&collection()), point_ids(&entries), "and holds every document, not only the new one"); - assert_eq!(projector.progress_status(), IndexStatus::Current); -} - -/// F2. A model re-pulled with other dimensions is seen before anything is -/// written: no vector of the new length is offered to the old collection, and -/// the next step rebuilds the collection to match. -#[test] -fn a_model_that_changes_dimensions_after_startup_rebuilds_the_collection() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - - embedder.state.lock().unwrap().identity.dimensions = 8; - projector.want(entries[3..].iter().cloned()); - let writes = index.state.lock().unwrap().upserts.len(); - assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the change is seen, not written through"); - assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "no vector of 8 was offered to a collection of 4"); - drain(&mut projector); - - let state = index.state.lock().unwrap(); - assert_eq!(state.recreated.len(), 2); - assert_eq!(state.recreated.last().unwrap().1.dimensions, 8); - let stored = &state.collections[&collection()]; - assert_eq!(stored.points.len(), 4); - assert!(stored.points.values().all(|point| point.vector.len() == 8)); -} - -/// F3. Two minds that share an instance name never share a collection: the -/// second is refused, and the first's points, label and history are untouched. -#[test] -fn a_second_mind_of_the_same_name_is_refused_and_leaves_the_first_alone() { - let (_first_root, first) = mind_with_documents(); - let (_second_root, second) = other_mind_of_the_same_name(); - let (first_id, second_id) = (first.genesis_receipt_id().unwrap().unwrap(), second.genesis_receipt_id().unwrap().unwrap()); - assert_ne!(first_id, second_id); - - let (embedder, index) = pair(); - let mut ours = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(first_id.clone())); - ours.want(first.index_entries(None).unwrap()); - drain(&mut ours); - let held = index.holding(&collection()); - assert_eq!(held.len(), 4); - - let intruder = FakeEmbedder::new("d1"); - let mut theirs = Projector::new(intruder.clone(), index.clone(), &slug(INSTANCE), Some(second_id)); - theirs.want(second.index_entries(None).unwrap()); - let error = theirs.advance().expect_err("another mind's collection is refused"); - assert!(matches!(error, StepError::Refused(_)), "{error}"); - assert!(matches!(theirs.failure_status(&error, 1), IndexStatus::Refused { .. })); - - let state = index.state.lock().unwrap(); - assert_eq!(state.recreated.len(), 1, "never rebuilt by the second mind"); - assert_eq!(state.upserts.len(), 1, "never written by the second mind"); - assert!(intruder.texts().is_empty()); - let Described::Labelled(label) = &state.collections[&collection()].label else { panic!() }; - assert_eq!(label.mind, first_id); - drop(state); - assert_eq!(index.holding(&collection()), held); -} - -/// A mind with no receipt has no identity: nothing is reconciled and no -/// collection is made, until the first admission reveals one. -#[test] -fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { - let root = tempfile::tempdir().unwrap(); - let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - assert_eq!(mind.genesis_receipt_id().unwrap(), None); - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); - assert_eq!(projector.progress_status(), IndexStatus::Current); - assert_eq!(projector.advance().unwrap(), Advance::Idle); - let identified = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - assert_eq!(identified.progress_status(), IndexStatus::Behind { pending: 0 }, "an identified mind has yet to build its first index: lag, not a rebuild"); - assert!(index.state.lock().unwrap().collections.is_empty()); - - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); - let mut daemon = Daemon::new(mind, sink); - let seed = campaign_seed(); - let without_text = vec![seed[0].clone(), seed[1].clone()]; - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, without_text)), now()).answered()); - for _ in 0..500 { - if index.state.lock().unwrap().collections.contains_key(&collection()) { - break; - } - std::thread::sleep(Duration::from_millis(10)); - } - let state = index.state.lock().unwrap(); - let Described::Labelled(label) = &state.collections[&collection()].label else { panic!("no collection was made") }; - assert_eq!(Some(&label.mind), daemon.mind().genesis_receipt_id().unwrap().as_ref()); -} - -/// Waits for a condition the worker will make true, without a clock in the -/// assertion: it polls for at most ten seconds and then says what never -/// happened. -fn eventually(what: &str, mut condition: impl FnMut() -> bool) { - for _ in 0..1000 { - if condition() { - return; - } - std::thread::sleep(Duration::from_millis(5)); - } - panic!("{what} never happened"); -} - -/// A model re-pulled under the same name and dimensions but a new digest must -/// not mix its vectors with the old model's. The check is before the write, -/// not after a failure: nothing is offered to the old collection, the -/// collection is rebuilt under the new digest, and every document is embedded -/// again. -#[test] -fn a_model_re_pulled_with_the_same_dimensions_never_writes_beside_the_old_vectors() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - - embedder.state.lock().unwrap().identity.digest = "d2".into(); - projector.want(entries[3..].iter().cloned()); - let writes = index.state.lock().unwrap().upserts.len(); - assert_eq!(projector.advance().unwrap(), Advance::Progressed); - assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "the new vector was not written into the old collection"); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 1 }, "the reconciliation was forgotten, not the entry"); - - drain(&mut projector); - let state = index.state.lock().unwrap(); - assert_eq!(state.recreated.len(), 2); - assert_eq!(state.recreated.last().unwrap().1.model_digest, "d2"); - let stored = &state.collections[&collection()]; - assert_eq!(stored.points.len(), 4, "every document is in the rebuilt collection"); - assert_eq!(embedder.texts().len(), 3 + 4, "and was embedded again under the new digest"); -} - -/// Nothing is written while idle, so no write can fail to announce that the -/// collection is gone or the model has changed. The idle worker looks again on -/// its own clock: a lost collection is refilled and a re-pulled model rebuilds -/// it, with no admission to prompt either. -#[test] -fn an_idle_worker_finds_a_lost_collection_and_a_changed_model_without_a_write() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let recheck = Backoff { recheck: Duration::from_millis(20), ..quick() }; - let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries.clone(), recheck); - sink.wait_status(|status| *status == IndexStatus::Current); - assert_eq!(index.holding(&collection()), point_ids(&entries)); - - index.state.lock().unwrap().collections.remove(&collection()); - eventually("the lost collection being made again and refilled", || { - index.state.lock().unwrap().recreated.len() == 2 && index.holding(&collection()) == point_ids(&entries) - }); - - embedder.state.lock().unwrap().identity.digest = "d2".into(); - eventually("the collection being rebuilt under the new digest", || { - let state = index.state.lock().unwrap(); - state.recreated.len() == 3 - && state.recreated.last().is_some_and(|(_, meta)| meta.model_digest == "d2") - && state.collections[&collection()].points.len() == 4 - }); - sink.wait_status(|status| *status == IndexStatus::Current); -} - -/// A search embeds the query behind the model's instruction, asks the index -/// for exactly `top_k` among the members it was handed (no oversample), and -/// hands back candidates by id and kind; a hit of a kind this organ has no -/// name for is left out. -#[test] -fn a_search_embeds_the_query_with_its_instruction_and_asks_for_top_k_among_the_members() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - index.state.lock().unwrap().hits = vec![ - Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }, - Hit { doc_id: "eureka-state:mystery:X".into(), kind: "mystery".into(), score: 0.8 }, - ]; - - let mut among: Vec = entries.iter().map(|entry| entry.id.id.0.clone()).collect(); - among.push("eureka-state:mystery:X".into()); - let hits = projector.search("who owns the state", 3, &among, SEARCH_DEADLINE).unwrap(); - assert_eq!(hits.len(), 1, "the unknown kind is left out"); - assert_eq!(hits[0].0.id.0, "eureka-state:question:Q1"); - assert_eq!(hits[0].0.kind, PipelineKind::Question); - assert_eq!(hits[0].1, 0.9); - let embedded = embedder.texts(); - assert_eq!(embedded.last().unwrap(), &format!("Instruct: {QUERY_INSTRUCTION}\nQuery: who owns the state")); - projector.search("again", 100, &among, SEARCH_DEADLINE).unwrap(); - let state = index.state.lock().unwrap(); - assert_eq!(state.searches.iter().map(|(name, length, limit)| (name.as_str(), *length, *limit)).collect::>(), [ - (collection().as_str(), 4, 3), - (collection().as_str(), 4, 100), - ], "top_k as asked, with no oversample"); - let points: Vec = among.iter().map(|id| point_id(id)).collect(); - assert_eq!(state.among, [points.clone(), points], "the index was handed the members' point ids"); -} - -/// A search is never run against a collection that is not known to fit the -/// model: before reconciliation, and after the model has changed, it refuses -/// by saying so rather than comparing vectors of two models. -#[test] -fn a_search_refuses_when_the_index_is_not_reconciled_with_the_model() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - assert!(format!("{:#}", fresh.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()).contains("not been built")); - assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); - - let mut projector = projected(&embedder, &index, &entries); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - let before = embedder.texts().len(); - assert!(format!("{:#}", projector.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()).contains("model changed")); - assert_eq!(embedder.texts().len(), before, "the query was not embedded under the new model"); - assert!(index.state.lock().unwrap().searches.is_empty()); - assert!(projector.stale(), "and the reconciliation is to be redone"); -} - -/// The source delete names the repo, the paths when given, and, when it keeps a -/// commit, excludes that commit's points: the filter is what spares the chunks -/// just written. -#[test] -fn qdrant_source_delete_filters_by_repo_paths_and_the_commit_it_keeps() { - use crate::sources::SourceIndex; - let deleting = stub(&[("POST /collections/", 200, r#"{"result":{"status":"completed"},"status":"ok"}"#)]); - let mut store = QdrantIndex::new(&deleting.url); - store.delete_by("Demo", Some(&["a.rs".to_owned()]), Some("abc123")).unwrap(); - store.delete_by("Gone", None, None).unwrap(); - let seen = deleting.seen.lock().unwrap(); - let bodies: Vec = seen.iter().map(|(_, _, body)| serde_json::from_str(body).unwrap()).collect(); - assert!(seen.iter().all(|(method, path, _)| method == "POST" && path.ends_with("/points/delete?wait=true"))); - assert_eq!( - bodies[0]["filter"], - serde_json::json!({ - "must": [{ "key": "repo", "match": { "value": "Demo" } }, { "key": "path", "match": { "any": ["a.rs"] } }], - "must_not": [{ "key": "commit", "match": { "value": "abc123" } }] - }) - ); - assert_eq!(bodies[1]["filter"], serde_json::json!({ "must": [{ "key": "repo", "match": { "value": "Gone" } }] })); -} - -/// The search scores exactly the points it is handed (a has_id filter, exact -/// search; which documents are members is the mind's), and reads back the -/// document id, the kind and the score, in the order Qdrant gave them. A hit -/// missing what the join needs is an error, not a guess. -#[test] -fn qdrant_search_scores_only_the_members_exactly_and_reads_ids_kinds_and_scores() { - let name = collection(); - let answering = stub(&[( - "POST /collections/", - 200, - r#"{"result":[{"id":"p1","score":0.75,"payload":{"doc_id":"eureka-state:question:Q1","kind":"question"}},{"id":"p2","score":0.5,"payload":{"doc_id":"eureka-state:ruling:R1","kind":"ruling"}}]}"#, - )]); - let hits = QdrantIndex::new(&answering.url).search(&name, &[0.5, 1.0, 2.0, 3.0], &["p1".to_owned(), "p2".to_owned()], 12, TIME).unwrap(); - assert_eq!( - hits, - [ - Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.75 }, - Hit { doc_id: "eureka-state:ruling:R1".into(), kind: "ruling".into(), score: 0.5 }, - ] - ); - let seen = answering.seen.lock().unwrap(); - assert_eq!((seen[0].0.as_str(), seen[0].1.as_str()), ("POST", format!("/collections/{name}/points/search").as_str())); +//! The projection's rules, against in-memory ports; the adapters' request and +//! answer shapes, against a stub that speaks just enough HTTP. No test needs a +//! live Qdrant or Ollama. + +use std::collections::BTreeSet; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, IndexStatus}; +use huginn_mind::{Mind, OwnedRedbMessagePackBackingStore, PipelineAdmissionOutcome}; +use tempfile::TempDir; + +use super::fakes::{FakeEmbedder, FakeIndex, Gate, Stored, pair}; +use super::ollama::OllamaEmbedder; +use super::qdrant::QdrantIndex; +use super::*; +use crate::daemon::Daemon; +use crate::daemon::tests::{INSTANCE, OTHER, batch, campaign_seed, now, question_and_ruling, slug}; + +type AfterAbsorb = Box; +type Edit = (&'static str, Box); + +thread_local! { + /// Runs once, in the worker's own turn, between the inbox being absorbed + /// and a search being taken, holding the guard `serve_search` holds. + static AFTER_ABSORB: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +} + +/// The hook `serve_search` calls at the point where a second lock once left a +/// gap. It is the test's only way in: nothing else can reach that point. +pub(super) fn after_absorb(guard: &mut Shared) { + if let Some(hook) = AFTER_ABSORB.with(|hook| hook.borrow_mut().take()) { + hook(guard); + } +} + +fn collection() -> String { + collection_name(&slug(INSTANCE)) +} + +/// The bound the adapter tests give a call. +const TIME: Duration = Duration::from_secs(30); + +/// A mind identity for tests that project entries without a mind behind them. +const MIND: &str = "mind-commit-first"; + +/// A mind of the same instance name whose first batch differs, so its genesis +/// receipt id does too. +fn other_mind_of_the_same_name() -> (TempDir, Mind) { + let root = tempfile::tempdir().unwrap(); + let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (question, _) = question_and_ruling(); + let mut first = campaign_seed(); + first.push(question); + let outcome = mind.admit(batch(INSTANCE, first), now()); + assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); + (root, mind) +} + +/// A real mind holding a campaign, a question, the ruling that answers it and +/// the resolution admission derived: four indexable documents, beside the +/// identity and the stewardship, which carry no text. +fn mind_with_documents() -> (TempDir, Mind) { + let root = tempfile::tempdir().unwrap(); + let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (question, ruling) = question_and_ruling(); + for documents in [campaign_seed(), vec![question, ruling]] { + let outcome = mind.admit(batch(INSTANCE, documents), now()); + assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); + } + (root, mind) +} + +/// How many times the collection was created and how many upserts it took. +fn counts(index: &FakeIndex) -> (usize, usize) { + let state = index.state.lock().unwrap(); + (state.recreated.len(), state.upserts.len()) +} + +/// Steps until the projector has nothing to do. A step that never settles is a +/// failure of the test, not a hang: it is bounded. +fn drain(projector: &mut Projector) { + for _ in 0..1000 { + if projector.advance().unwrap() == Advance::Idle { + return; + } + } + panic!("the projector was still stepping after 1000 steps"); +} + +fn point_ids(entries: &[IndexEntry]) -> BTreeSet { + entries.iter().map(|entry| point_id(&entry.id.id.0)).collect() +} + +fn projected(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + drain(&mut projector); + projector +} + +/// A projector whose collection was lost while it ran and has been made again: +/// it owes every entry, and is rebuilding. +fn rebuilding(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { + let mut projector = projected(embedder, index, &entries[..1]); + index.state.lock().unwrap().collections.remove(&collection()); + projector.want(entries[1..].iter().cloned()); + projector.advance().expect_err("the write meets no collection"); + assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the reconciliation makes the collection again"); + projector +} + +/// P4's function, as voidbot's `toQdrantPointId` computes it: the first sixteen +/// bytes of the SHA-256, version and variant bits set. The fixed vectors were +/// computed independently of this crate. +#[test] +fn point_ids_are_a_pure_function_of_the_document_id() { + assert_eq!(point_id("eureka-state:question:Q1"), "1e8a661b-4b7e-5245-a9ad-b15a3fa27031"); + assert_eq!(point_id("eureka-state:campaign:self"), "d4135998-051d-57fb-9360-82964874fbc9"); + assert_eq!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q1")); + assert_ne!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q2")); +} + +/// The projection converges by set difference: after a crash that lost two +/// points, a restart embeds and writes those two, and nothing else, and the +/// collection is not rebuilt. +#[test] +fn reconcile_indexes_exactly_the_indexable_documents_the_collection_lacks() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + assert_eq!(entries.len(), 4); + + let (embedder, index) = pair(); + projected(&embedder, &index, &entries); + assert_eq!( + index.holding(&collection()), + point_ids(&entries), + "one point per indexable document, none for the identity or the stewardship" + ); + + let lost: Vec = entries.iter().take(2).cloned().collect(); + { + let mut state = index.state.lock().unwrap(); + let stored = state.collections.get_mut(&collection()).unwrap(); + for entry in &lost { + stored.points.remove(&point_id(&entry.id.id.0)); + } + state.upserts.clear(); + state.recreated.clear(); + } + + let restarted = FakeEmbedder::new("d1"); + projected(&restarted, &index, &entries); + let state = index.state.lock().unwrap(); + assert!(state.recreated.is_empty(), "a collection that agrees is not rebuilt"); + let written: BTreeSet = state.upserts.concat().into_iter().collect(); + assert_eq!(written, point_ids(&lost), "exactly the missing points were written"); + assert_eq!(state.upserts.concat().len(), 2, "each once"); + assert_eq!(restarted.texts().len(), 2, "and only their text was embedded"); + drop(state); + assert_eq!(index.holding(&collection()), point_ids(&entries)); +} + +/// Compatibility is the model by name and digest, its dimensions and the text +/// version. Any of them differing rebuilds the collection whole, and no vector +/// of one model survives beside another's. +#[test] +fn a_model_digest_change_rebuilds_the_collection() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let changes: [Edit; 3] = [ + ("digest", Box::new(|identity| identity.digest = "d2".into())), + ("name", Box::new(|identity| identity.name = "other-model".into())), + ("dimensions", Box::new(|identity| identity.dimensions = 8)), + ]; + for (what, change) in changes { + let (embedder, index) = pair(); + projected(&embedder, &index, &entries); + assert_eq!(index.state.lock().unwrap().recreated.len(), 1, "the first run creates the collection"); + + let second = FakeEmbedder::new("d1"); + change(&mut second.state.lock().unwrap().identity); + let expected = second.state.lock().unwrap().identity.clone(); + projected(&second, &index, &entries); + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2, "a changed {what} rebuilds"); + let (_, meta) = state.recreated.last().unwrap(); + assert_eq!( + (&meta.model, &meta.model_digest, meta.dimensions), + (&expected.name, &expected.digest, expected.dimensions), + "{what}" + ); + assert_eq!(second.texts().len(), 4, "every document is embedded again under {what}"); + assert_eq!(state.collections[&collection()].points.len(), 4); + } + + // The same model, digest and dimensions: nothing is rebuilt or rewritten. + let (embedder, index) = pair(); + projected(&embedder, &index, &entries); + let same = FakeEmbedder::new("d1"); + projected(&same, &index, &entries); + assert_eq!(index.state.lock().unwrap().recreated.len(), 1); + assert!(same.texts().is_empty()); + + // A text version the collection was not built with is a rebuild too. + { + let mut state = index.state.lock().unwrap(); + let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; + meta.index_text_version += 1; + } + projected(&FakeEmbedder::new("d1"), &index, &entries); + assert_eq!(index.state.lock().unwrap().recreated.len(), 2); +} + +/// No cross-mind writes: a collection that another writer or another instance +/// made, or that carries no label, is left exactly as it was. +#[test] +fn a_collection_owned_by_another_instance_is_never_recreated() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let ours = |managed_by: &str, instance: &str, mind: &str| CollectionMeta { + managed_by: managed_by.into(), + instance: instance.into(), + mind: mind.into(), + model: "model".into(), + model_digest: "d1".into(), + dimensions: 4, + index_text_version: INDEX_TEXT_VERSION, + }; + for foreign in [ + Described::Unlabelled, + Described::Labelled(ours(MANAGED_BY, OTHER, MIND)), + Described::Labelled(ours("voidbot", INSTANCE, MIND)), + Described::Labelled(ours(MANAGED_BY, INSTANCE, "mind-commit-another")), + Described::Labelled(ours(MANAGED_BY, INSTANCE, "")), + ] { + let (embedder, index) = pair(); + index.plant(&collection(), foreign.clone()); + let sentinel = point_id("someone-elses-document"); + index.state.lock().unwrap().collections.get_mut(&collection()).unwrap().points.insert( + sentinel.clone(), + Point { + id: sentinel.clone(), + vector: vec![0.0; 4], + payload: PointPayload { + doc_id: "someone-elses-document".into(), + kind: "x".into(), + root: "x".into(), + ordinal: 1, + text_sha256: "x".into(), + }, + }, + ); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + let error = projector.reconcile().expect_err("a foreign collection is refused"); + assert!(matches!(error, StepError::Refused(_)), "{foreign:?}: {error}"); + assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Refused { pending: 4, .. })); + let state = index.state.lock().unwrap(); + assert!(state.recreated.is_empty() && state.upserts.is_empty(), "{foreign:?} was written to"); + assert_eq!(state.collections[&collection()].label, foreign); + assert!(state.collections[&collection()].points.contains_key(&sentinel), "{foreign:?} lost a point"); + assert!(embedder.texts().is_empty()); + } +} + +/// The vector's length is checked against the collection's, so a flush that +/// the model did not change is still refused when the embedder answers wrongly. +#[test] +fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + projector.reconcile().unwrap(); + embedder.state.lock().unwrap().vector_length = Some(8); + let error = projector.flush_batch().expect_err("a longer vector is refused"); + assert!(matches!(error, StepError::Unavailable(_)), "{error}"); + assert_eq!(projector.pending(), 4); + assert!(index.state.lock().unwrap().upserts.is_empty()); +} + +/// A flush embeds through the same check as a search: a model that changed +/// before the embed is never asked, a model that changed during it has its +/// vectors dropped, and a collection that is no longer this mind's is refused. +/// In each case nothing is written, the documents stay wanted, and the +/// reconciliation is forgotten. +#[test] +fn a_flush_is_refused_as_a_search_is_when_the_model_or_the_collection_changed() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let reconciled = || { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + projector.reconcile().unwrap(); + (embedder, index, projector) + }; + + let (embedder, index, mut projector) = reconciled(); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + projector.flush_batch().expect("a changed model is not a failure"); + assert!(embedder.texts().is_empty(), "before: the texts were not embedded under the new model"); + assert!(index.state.lock().unwrap().upserts.is_empty()); + assert_eq!((projector.pending(), projector.stale()), (4, true), "before"); + + let (embedder, index, mut projector) = reconciled(); + change_label(&index, |meta| meta.mind = "mind-commit-another".into()); + let error = projector.flush_batch().expect_err("another mind's label is refused"); + assert!(error.to_string().contains("no longer the one this mind's index was built as"), "{error}"); + assert!(embedder.texts().is_empty(), "label: nothing was embedded"); + assert!(index.state.lock().unwrap().upserts.is_empty()); + assert_eq!((projector.pending(), projector.stale()), (4, true), "label"); + + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let index = FakeIndex::default(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + projector.reconcile().unwrap(); + let flushing = std::thread::spawn(move || { + let flushed = projector.flush_batch(); + (projector, flushed) + }); + gate.wait_arrived(1); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + gate.open(); + let (projector, flushed) = flushing.join().unwrap(); + flushed.expect("a changed model is not a failure"); + assert!(index.state.lock().unwrap().upserts.is_empty(), "during: the vectors were not written"); + assert_eq!((projector.pending(), projector.stale()), (4, true), "during"); +} + +fn change_label(index: &FakeIndex, change: impl FnOnce(&mut CollectionMeta)) { + let mut state = index.state.lock().unwrap(); + let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; + change(meta); +} +/// Every call on the search path is bounded by what remains of the search's own +/// time: the identity reads, the label read, the embed and the query. A search +/// whose embed cannot be recalled therefore cannot hold the writes off past the +/// moment nobody is waiting for it. A bulk flush keeps its own, longer, bound. +#[test] +fn every_call_on_the_search_path_is_bounded_by_the_searchs_time_and_the_flush_by_its_own() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + { + let state = embedder.state.lock().unwrap(); + assert!(!state.bounds.is_empty()); + let near = |bound: &Duration| *bound <= BULK_EMBED_TIMEOUT && *bound > BULK_EMBED_TIMEOUT - Duration::from_secs(60); + assert!(state.bounds.iter().all(near), "the flushes' embeds: {:?}", state.bounds); + assert!(state.identity_bounds.iter().all(near), "the reconciliation's and the flushes' identity reads: {:?}", state.identity_bounds); + } + embedder.state.lock().unwrap().bounds.clear(); + embedder.state.lock().unwrap().identity_bounds.clear(); + index.state.lock().unwrap().bounds.clear(); + + let within = Duration::from_secs(20); + projector.search("anything", 3, &[], within).unwrap(); + let embedder = embedder.state.lock().unwrap(); + let index = index.state.lock().unwrap(); + assert_eq!(embedder.identity_bounds.len(), 2, "the model is read before and after the embed"); + assert_eq!(embedder.bounds.len(), 1); + assert_eq!(index.bounds.iter().map(|(call, _)| *call).collect::>(), ["describe", "search"]); + let bounds: Vec = embedder + .identity_bounds + .iter() + .chain(&embedder.bounds) + .copied() + .chain(index.bounds.iter().map(|(_, bound)| *bound)) + .collect(); + assert!(bounds.iter().all(|bound| !bound.is_zero() && *bound <= within), "{bounds:?}"); + assert_eq!(Backoff::default().search_deadline, SEARCH_DEADLINE); + assert!(SEARCH_DEADLINE < BULK_EMBED_TIMEOUT); +} + +/// A hung Ollama does not hold the worker past the search that asked: the +/// identity read never answers, and the search is answered, and the worker +/// free, when the search's own time has passed. A search that waited past its +/// time in the queue is answered without asking the embedder anything. A +/// ticket's time is stamped by the sink (see `the_sink_owns_the_searchs_deadline_and_tells_the_loop`). +#[test] +fn a_hung_embedder_holds_the_worker_no_longer_than_the_search_that_asked() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + + ask_sink(&mut sink, "the ticket's time", 3).unwrap(); + answers(&mut sink, 1); + + embedder.state.lock().unwrap().hang_identity = true; + let started = Instant::now(); + let due = started + Duration::from_millis(400); + lock(&sink.shared).searches.push_back(Queued { ticket: SearchTicket(70), text: "hung".into(), top_k: 3, among: vec![], due }); + sink.shared.1.notify_all(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, SearchTicket(70)); + assert!(answered[0].1.is_err(), "{answered:?}"); + assert!(started.elapsed() < Duration::from_secs(5), "the worker was held for {:?}", started.elapsed()); + + embedder.state.lock().unwrap().hang_identity = false; + let reads = embedder.state.lock().unwrap().identity_bounds.len(); + lock(&sink.shared).searches.push_back(Queued { + ticket: SearchTicket(71), + text: "late".into(), + top_k: 3, + among: vec![], + due: Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now), + }); + sink.shared.1.notify_all(); + let late = answers(&mut sink, 1); + let Err(why) = &late[0].1 else { panic!("{late:?}") }; + assert!(why.contains("has run out"), "{why}"); + assert_eq!(embedder.state.lock().unwrap().identity_bounds.len(), reads, "nothing was asked for a search nobody waits for"); + + let ticket = ask_sink(&mut sink, "the worker is free", 3).expect("the worker is still current"); + assert_eq!(answers(&mut sink, 1)[0].0, ticket); +} + +/// The Ollama adapter enforces the bound it is given: an embed that is never +/// answered fails at the bound, not at the adapter's default. +#[test] +fn an_ollama_embed_that_is_never_answered_fails_at_its_bound() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let held = std::thread::spawn(move || { + let mut held = Vec::new(); + for stream in listener.incoming().take(1) { + held.push(stream.unwrap()); + std::thread::sleep(Duration::from_secs(8)); + } + }); + let (sent, received) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut embedder = OllamaEmbedder::new(&url, "m:1"); + sent.send(embedder.embed(&["a".into()], Duration::from_millis(300)).is_err()).ok(); + }); + assert_eq!(received.recv_timeout(Duration::from_secs(5)), Ok(true), "the embed did not fail at its bound"); + drop(held); +} +fn whoami_index(daemon: &mut Daemon) -> IndexStatus { + match daemon.handle(HuginnMindRequest::Whoami, now()).answered() { + HuginnMindResponse::Whoami(status) => status.index, + other => panic!("expected a status, got {other:?}"), + } +} + +fn committed(response: HuginnMindResponse) { + assert!( + matches!(response, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), + "{response:?}" + ); +} + +fn quick() -> Backoff { + Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) } +} + +/// The index never delays or refuses admission: the embedder is shut, the +/// worker is inside it, and two more admissions are answered at once. The gate +/// counts arrivals, so this observes where the work is and not how long it +/// took. +#[test] +fn an_admission_replies_while_the_embedder_is_blocked() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let index = FakeIndex::default(); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), mind.index_entries(None).unwrap(), quick()); + let mut daemon = Daemon::new(mind, sink); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, campaign_seed())), now()).answered()); + gate.wait_arrived(1); + let (question, ruling) = question_and_ruling(); + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()).answered()); + assert_eq!( + whoami_index(&mut daemon), + IndexStatus::Behind { pending: 4 }, + "the campaign is in the embedder and three more are queued" + ); + assert!(index.holding(&collection()).is_empty(), "nothing has been written yet"); + + gate.open(); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 4); + assert_eq!(whoami_index(&mut daemon), IndexStatus::Current); +} + +/// A failing embedder is reported by `whoami`, retried on the backoff, and +/// nothing is dropped: when it returns, every document is written. +#[test] +fn a_failing_embedder_is_visible_in_whoami_and_retried_never_dropped() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (embedder, index) = pair(); + let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); + let mut daemon = Daemon::new(mind, sink); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + + embedder.state.lock().unwrap().down = true; + let (question, ruling) = question_and_ruling(); + for documents in [campaign_seed(), vec![question, ruling]] { + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), now()).answered()); + } + let seen = daemon.index().wait_status(|status| matches!(status, IndexStatus::Failing { attempts, .. } if *attempts >= 2)); + let IndexStatus::Failing { pending, error, .. } = seen else { panic!() }; + assert_eq!(pending, 4); + assert!(error.contains("the embedder is down"), "{error}"); + assert!(matches!(whoami_index(&mut daemon), IndexStatus::Failing { .. })); + assert!(index.holding(&collection()).is_empty()); + + embedder.state.lock().unwrap().down = false; + daemon.index().wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 4, "every admitted document was kept and written"); +} + +/// A collection that is not this mind's is reported as refused, never +/// recreated, and re-read on the backoff: once it is gone the projection +/// completes. +#[test] +fn a_refused_collection_is_reported_and_reread() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + index.plant(&collection(), Described::Unlabelled); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries, quick()); + let mut daemon = Daemon::new(mind, sink); + daemon.index().wait_status(|status| matches!(status, IndexStatus::Refused { attempts, .. } if *attempts >= 2)); + assert!(matches!(whoami_index(&mut daemon), IndexStatus::Refused { pending: 4, .. })); + assert!(index.state.lock().unwrap().recreated.is_empty()); + + index.state.lock().unwrap().collections.remove(&collection()); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 4); +} + +// The adapters, against a stub that answers the routes it is given. + +struct Stub { + url: String, + seen: Arc>>, +} + +type Route = (&'static str, u16, &'static str); + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|window| window == needle) +} + +/// Serves each route once, in order, then repeats the last route that matched. +/// A route matches `" "` by prefix. +fn stub(routes: &[Route]) -> Stub { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let seen = Arc::new(Mutex::new(Vec::new())); + let routes: Arc>> = Arc::new(Mutex::new(routes.iter().map(|route| (*route, false)).collect())); + let log = Arc::clone(&seen); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { break }; + let (routes, log) = (Arc::clone(&routes), Arc::clone(&log)); + std::thread::spawn(move || answer_connection(stream, &routes, &log)); + } + }); + Stub { url, seen } +} + +fn answer_connection( + mut stream: TcpStream, + routes: &Mutex>, + log: &Mutex>, +) { + let mut buffer: Vec = Vec::new(); + loop { + let header_end = loop { + if let Some(at) = find(&buffer, b"\r\n\r\n") { + break at + 4; + } + let mut chunk = [0_u8; 4096]; + match stream.read(&mut chunk) { + Ok(0) | Err(_) => return, + Ok(read) => buffer.extend_from_slice(&chunk[..read]), + } + }; + let head = String::from_utf8_lossy(&buffer[..header_end]).into_owned(); + let length = head + .lines() + .find_map(|line| line.to_ascii_lowercase().strip_prefix("content-length:").map(|value| value.trim().parse::().unwrap())) + .unwrap_or(0); + while buffer.len() < header_end + length { + let mut chunk = [0_u8; 4096]; + match stream.read(&mut chunk) { + Ok(0) | Err(_) => return, + Ok(read) => buffer.extend_from_slice(&chunk[..read]), + } + } + let body = String::from_utf8_lossy(&buffer[header_end..header_end + length]).into_owned(); + buffer.drain(..header_end + length); + let mut first = head.lines().next().unwrap().split(' '); + let (method, path) = (first.next().unwrap().to_owned(), first.next().unwrap().to_owned()); + log.lock().unwrap().push((method.clone(), path.clone(), body)); + let key = format!("{method} {path}"); + let (status, answer) = { + let mut routes = routes.lock().unwrap(); + let matching: Vec = + routes.iter().enumerate().filter(|(_, ((prefix, ..), _))| key.starts_with(prefix)).map(|(at, _)| at).collect(); + let chosen = matching.iter().copied().find(|at| !routes[*at].1).or(matching.last().copied()); + match chosen { + Some(at) => { + routes[at].1 = true; + (routes[at].0.1, routes[at].0.2) + } + None => (404, "{}"), + } + }; + let response = + format!("HTTP/1.1 {status} X\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{answer}", answer.len()); + if stream.write_all(response.as_bytes()).is_err() { + return; + } + } +} + +const TAGS: &str = r#"{"models":[{"name":"other:1","model":"other:1","digest":"zzz"},{"name":"m:1","model":"m:1","digest":"ac6d"}]}"#; +const SHOW: &str = r#"{"model_info":{"general.architecture":"qwen3","qwen3.embedding_length":4}}"#; +const EMBED: &str = r#"{"embeddings":[[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0]]}"#; + +#[test] +fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { + let ollama = stub(&[("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]); + let mut embedder = OllamaEmbedder::new(&ollama.url, "m:1"); + assert_eq!(embedder.model_identity(TIME).unwrap(), ModelIdentity { name: "m:1".into(), digest: "ac6d".into(), dimensions: 4 }); + let vectors = embedder.embed(&["a".into(), "b".into(), "c".into(), "d".into()], Duration::from_secs(5)).unwrap(); + assert_eq!(vectors.len(), 4); + assert_eq!(vectors[0], vec![0.5, 1.0, 2.0, 3.0]); + let seen = ollama.seen.lock().unwrap(); + let (_, _, body) = seen.iter().find(|(_, path, _)| path == "/api/embed").unwrap(); + assert!(body.contains("\"model\":\"m:1\"") && body.contains("\"input\":[\"a\",\"b\",\"c\",\"d\"]"), "{body}"); + drop(seen); + + // A listing entry that carries only `name` is still the model. + let bare = stub(&[ + ("GET /api/tags", 200, r#"{"models":[{"name":"m:1","digest":"ac6d"}]}"#), + ("POST /api/show", 200, SHOW), + ]); + assert_eq!(OllamaEmbedder::new(&bare.url, "m:1").model_identity(TIME).unwrap().digest, "ac6d"); + + let mut absent = OllamaEmbedder::new(&ollama.url, "not-pulled:1"); + assert!(format!("{:#}", absent.model_identity(TIME).unwrap_err()).contains("does not list the model")); + let refusing = stub(&[("POST /api/embed", 500, r#"{"error":"boom"}"#)]); + assert!(OllamaEmbedder::new(&refusing.url, "m:1").embed(&["a".into()], Duration::from_secs(5)).is_err()); +} + +/// Compatibility never reads the address: the same model at another address +/// leaves the collection alone, and the same address serving another digest +/// rebuilds it. +#[test] +fn the_embedders_address_is_not_part_of_the_collections_compatibility() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let routes = [("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]; + let repulled = TAGS.replace("ac6d", "beef"); + let index = FakeIndex::default(); + + let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); + first.want(entries.iter().cloned()); + drain(&mut first); + assert_eq!(counts(&index), (1, 1)); + + // Another address, the same model and digest. + let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); + moved.want(entries.iter().cloned()); + drain(&mut moved); + assert_eq!(counts(&index), (1, 1)); + + // The same model name re-pulled: another digest. + let leaked: &'static str = Box::leak(repulled.into_boxed_str()); + let other = [("GET /api/tags", 200, leaked), routes[1], routes[2]]; + let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); + rebuilt.want(entries.iter().cloned()); + drain(&mut rebuilt); + assert_eq!(counts(&index), (2, 2)); +} + +#[test] +fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { + let name = collection(); + let absent = stub(&[("GET /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); + assert_eq!(QdrantIndex::new(&absent.url).describe(&name, TIME).unwrap(), Described::Absent); + + let unlabelled = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":null}}}"#)]); + assert_eq!(QdrantIndex::new(&unlabelled.url).describe(&name, TIME).unwrap(), Described::Unlabelled); + let voidbot = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":{"managedBy":"voidbot"}}}}"#)]); + assert_eq!(QdrantIndex::new(&voidbot.url).describe(&name, TIME).unwrap(), Described::Unlabelled); + + let labelled = stub(&[( + "GET /collections/", + 200, + r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","mind":"mind-commit-first","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, + )]); + assert_eq!( + QdrantIndex::new(&labelled.url).describe(&name, TIME).unwrap(), + Described::Labelled(CollectionMeta { + managed_by: "huginn".into(), + instance: "yggdrasil".into(), + mind: "mind-commit-first".into(), + model: "m:1".into(), + model_digest: "ac6d".into(), + dimensions: 4, + index_text_version: 1, + }) + ); + + // A scroll is followed to its last page. + let paged = stub(&[ + ("POST /collections/", 200, r#"{"result":{"points":[{"id":"a"},{"id":"b"}],"next_page_offset":"b"}}"#), + ("POST /collections/", 200, r#"{"result":{"points":[{"id":"c"}],"next_page_offset":null}}"#), + ]); + let ids = QdrantIndex::new(&paged.url).ids(&name).unwrap(); + let expected: BTreeSet = ["a", "b", "c"].map(String::from).into_iter().collect(); + assert_eq!(ids, expected); + let seen = paged.seen.lock().unwrap(); + assert!(!seen[0].2.contains("offset") && seen[1].2.contains("\"offset\":\"b\""), "{seen:?}"); + drop(seen); + + // A rebuild drops, then creates with the metadata and the vector shape. + let writing = stub(&[("DELETE /collections/", 200, "{}"), ("PUT /collections/", 200, "{}")]); + let meta = CollectionMeta { + managed_by: MANAGED_BY.into(), + instance: INSTANCE.into(), + mind: MIND.into(), + model: "m:1".into(), + model_digest: "ac6d".into(), + dimensions: 4, + index_text_version: INDEX_TEXT_VERSION, + }; + let mut store = QdrantIndex::new(&writing.url); + store.recreate(&name, &meta).unwrap(); + let point = Point { + id: point_id("d"), + vector: vec![1.0, 2.0, 3.0, 4.0], + payload: PointPayload { doc_id: "d".into(), kind: "question".into(), root: "eureka-state".into(), ordinal: 7, text_sha256: "h".into() }, + }; + store.upsert(&name, &[point]).unwrap(); + let seen = writing.seen.lock().unwrap(); + let paths: Vec = seen.iter().map(|(method, path, _)| format!("{method} {path}")).collect(); + assert_eq!( + paths, + [ + format!("DELETE /collections/{name}?wait=true"), + format!("PUT /collections/{name}?wait=true"), + format!("PUT /collections/{name}/points?wait=true"), + ] + ); + for expected in ["\"size\":4", "\"distance\":\"Cosine\"", "\"on_disk\":true", "\"managed_by\":\"huginn\"", "\"index_text_version\":1"] { + assert!(seen[1].2.contains(expected), "{expected} in {}", seen[1].2); + } + for expected in ["\"doc_id\":\"d\"", "\"kind\":\"question\"", "\"root\":\"eureka-state\"", "\"ordinal\":7", "\"text_sha256\":\"h\""] { + assert!(seen[2].2.contains(expected), "{expected} in {}", seen[2].2); + } + + let failing = stub(&[("PUT /collections/", 500, r#"{"status":{"error":"disk full"}}"#)]); + assert!(QdrantIndex::new(&failing.url).upsert(&name, &[]).is_err()); +} + +/// The collection's name is the contract with the store: one per instance, +/// `huginn_mind_`. +#[test] +fn the_collection_is_named_for_the_instance() { + assert_eq!(collection_name(&slug("yggdrasil")), "huginn_mind_yggdrasil"); + assert_eq!(collection_name(&slug("a.b")), "huginn_mind_a.b"); +} + +#[test] +fn a_step_error_says_what_stopped_it() { + assert_eq!(StepError::Refused("not ours".into()).to_string(), "not ours"); + let unavailable = StepError::Unavailable(anyhow::anyhow!("inner").context("outer")); + assert_eq!(unavailable.to_string(), "outer: inner"); +} + +/// What the sink reports is the worker's health with the inbox added to the +/// pending count, and a current index with something queued is behind. +#[test] +fn queued_entries_count_toward_every_status_that_has_a_pending_count() { + use IndexStatus as S; + assert_eq!(including(&S::Current, 0), S::Current); + assert_eq!(including(&S::Current, 3), S::Behind { pending: 3 }); + assert_eq!(including(&S::Reconciling { pending: 2 }, 3), S::Reconciling { pending: 5 }); + assert_eq!(including(&S::Behind { pending: 2 }, 3), S::Behind { pending: 5 }); + assert_eq!( + including(&S::Failing { pending: 2, attempts: 4, error: "e".into() }, 3), + S::Failing { pending: 5, attempts: 4, error: "e".into() } + ); + assert_eq!( + including(&S::Refused { pending: 2, attempts: 4, reason: "r".into() }, 3), + S::Refused { pending: 5, attempts: 4, reason: "r".into() } + ); +} + +/// Dropping the sink stops the worker: the worker's hold on the shared state +/// is released. A worker that never stopped would hold it forever. +#[test] +fn dropping_the_sink_stops_the_worker() { + let (embedder, index) = pair(); + let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let shared = Arc::downgrade(&sink.shared); + drop(sink); + eventually("the worker letting go of the shared state", || shared.upgrade().is_none()); +} + +/// F1. The promised curve, pure: 30 s doubling to a 10 min cap, never below +/// the first delay and never past the cap. A retry that shrank would hot-loop +/// against a dead Ollama. +#[test] +fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() { + let backoff = Backoff::default(); + assert_eq!((backoff.initial, backoff.max), (Duration::from_secs(30), Duration::from_secs(600))); + assert_eq!(backoff.recheck, Duration::from_secs(60), "an idle worker verifies the model and the collection each minute"); + let mut delay = backoff.initial; + let mut seen = vec![delay.as_secs()]; + for _ in 0..7 { + let next = backoff.after(delay); + assert!(next >= delay && next >= backoff.initial, "the delay never shrinks: {delay:?} -> {next:?}"); + assert!(next <= backoff.max, "the delay never passes the cap: {next:?}"); + delay = next; + seen.push(delay.as_secs()); + } + assert_eq!(seen, [30, 60, 120, 240, 480, 600, 600, 600]); + let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) }; + assert_eq!(small.after(Duration::from_millis(1)), Duration::from_millis(2)); + assert_eq!(small.after(Duration::from_millis(2)), Duration::from_millis(3)); +} + +/// F2. A collection lost while the daemon runs is found by the next failed +/// write, recreated and refilled from what the projector knows, without a +/// restart. +#[test] +fn a_collection_lost_after_startup_is_recreated_and_refilled() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 3); + + index.state.lock().unwrap().collections.remove(&collection()); + projector.want(entries[3..].iter().cloned()); + let error = projector.advance().expect_err("the write meets no collection"); + assert!(matches!(error, StepError::Unavailable(_)), "{error}"); + assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Failing { pending: 1, .. })); + + drain(&mut projector); + assert_eq!(index.state.lock().unwrap().recreated.len(), 2, "the collection was made again"); + assert_eq!(index.holding(&collection()), point_ids(&entries), "and holds every document, not only the new one"); + assert_eq!(projector.progress_status(), IndexStatus::Current); +} + +/// F2. A model re-pulled with other dimensions is seen before anything is +/// written: no vector of the new length is offered to the old collection, and +/// the next step rebuilds the collection to match. +#[test] +fn a_model_that_changes_dimensions_after_startup_rebuilds_the_collection() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + + embedder.state.lock().unwrap().identity.dimensions = 8; + projector.want(entries[3..].iter().cloned()); + let writes = index.state.lock().unwrap().upserts.len(); + assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the change is seen, not written through"); + assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "no vector of 8 was offered to a collection of 4"); + drain(&mut projector); + + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2); + assert_eq!(state.recreated.last().unwrap().1.dimensions, 8); + let stored = &state.collections[&collection()]; + assert_eq!(stored.points.len(), 4); + assert!(stored.points.values().all(|point| point.vector.len() == 8)); +} + +/// F3. Two minds that share an instance name never share a collection: the +/// second is refused, and the first's points, label and history are untouched. +#[test] +fn a_second_mind_of_the_same_name_is_refused_and_leaves_the_first_alone() { + let (_first_root, first) = mind_with_documents(); + let (_second_root, second) = other_mind_of_the_same_name(); + let (first_id, second_id) = (first.genesis_receipt_id().unwrap().unwrap(), second.genesis_receipt_id().unwrap().unwrap()); + assert_ne!(first_id, second_id); + + let (embedder, index) = pair(); + let mut ours = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(first_id.clone())); + ours.want(first.index_entries(None).unwrap()); + drain(&mut ours); + let held = index.holding(&collection()); + assert_eq!(held.len(), 4); + + let intruder = FakeEmbedder::new("d1"); + let mut theirs = Projector::new(intruder.clone(), index.clone(), &slug(INSTANCE), Some(second_id)); + theirs.want(second.index_entries(None).unwrap()); + let error = theirs.advance().expect_err("another mind's collection is refused"); + assert!(matches!(error, StepError::Refused(_)), "{error}"); + assert!(matches!(theirs.failure_status(&error, 1), IndexStatus::Refused { .. })); + + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 1, "never rebuilt by the second mind"); + assert_eq!(state.upserts.len(), 1, "never written by the second mind"); + assert!(intruder.texts().is_empty()); + let Described::Labelled(label) = &state.collections[&collection()].label else { panic!() }; + assert_eq!(label.mind, first_id); + drop(state); + assert_eq!(index.holding(&collection()), held); +} + +/// A mind with no receipt has no identity: nothing is reconciled and no +/// collection is made, until the first admission reveals one. +#[test] +fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + assert_eq!(mind.genesis_receipt_id().unwrap(), None); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert_eq!(projector.advance().unwrap(), Advance::Idle); + let identified = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + assert_eq!(identified.progress_status(), IndexStatus::Behind { pending: 0 }, "an identified mind has yet to build its first index: lag, not a rebuild"); + assert!(index.state.lock().unwrap().collections.is_empty()); + + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); + let mut daemon = Daemon::new(mind, sink); + let seed = campaign_seed(); + let without_text = vec![seed[0].clone(), seed[1].clone()]; + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, without_text)), now()).answered()); + for _ in 0..500 { + if index.state.lock().unwrap().collections.contains_key(&collection()) { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let state = index.state.lock().unwrap(); + let Described::Labelled(label) = &state.collections[&collection()].label else { panic!("no collection was made") }; + assert_eq!(Some(&label.mind), daemon.mind().genesis_receipt_id().unwrap().as_ref()); +} + +/// Waits for a condition the worker will make true, without a clock in the +/// assertion: it polls for at most ten seconds and then says what never +/// happened. +fn eventually(what: &str, mut condition: impl FnMut() -> bool) { + for _ in 0..1000 { + if condition() { + return; + } + std::thread::sleep(Duration::from_millis(5)); + } + panic!("{what} never happened"); +} + +/// A model re-pulled under the same name and dimensions but a new digest must +/// not mix its vectors with the old model's. The check is before the write, +/// not after a failure: nothing is offered to the old collection, the +/// collection is rebuilt under the new digest, and every document is embedded +/// again. +#[test] +fn a_model_re_pulled_with_the_same_dimensions_never_writes_beside_the_old_vectors() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + + embedder.state.lock().unwrap().identity.digest = "d2".into(); + projector.want(entries[3..].iter().cloned()); + let writes = index.state.lock().unwrap().upserts.len(); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "the new vector was not written into the old collection"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 1 }, "the reconciliation was forgotten, not the entry"); + + drain(&mut projector); + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2); + assert_eq!(state.recreated.last().unwrap().1.model_digest, "d2"); + let stored = &state.collections[&collection()]; + assert_eq!(stored.points.len(), 4, "every document is in the rebuilt collection"); + assert_eq!(embedder.texts().len(), 3 + 4, "and was embedded again under the new digest"); +} + +/// Nothing is written while idle, so no write can fail to announce that the +/// collection is gone or the model has changed. The idle worker looks again on +/// its own clock: a lost collection is refilled and a re-pulled model rebuilds +/// it, with no admission to prompt either. +#[test] +fn an_idle_worker_finds_a_lost_collection_and_a_changed_model_without_a_write() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let recheck = Backoff { recheck: Duration::from_millis(20), ..quick() }; + let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries.clone(), recheck); + sink.wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()), point_ids(&entries)); + + index.state.lock().unwrap().collections.remove(&collection()); + eventually("the lost collection being made again and refilled", || { + index.state.lock().unwrap().recreated.len() == 2 && index.holding(&collection()) == point_ids(&entries) + }); + + embedder.state.lock().unwrap().identity.digest = "d2".into(); + eventually("the collection being rebuilt under the new digest", || { + let state = index.state.lock().unwrap(); + state.recreated.len() == 3 + && state.recreated.last().is_some_and(|(_, meta)| meta.model_digest == "d2") + && state.collections[&collection()].points.len() == 4 + }); + sink.wait_status(|status| *status == IndexStatus::Current); +} + +/// A search embeds the query behind the model's instruction, asks the index +/// for exactly `top_k` among the members it was handed (no oversample), and +/// hands back candidates by id and kind; a hit of a kind this organ has no +/// name for is left out. +#[test] +fn a_search_embeds_the_query_with_its_instruction_and_asks_for_top_k_among_the_members() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + index.state.lock().unwrap().hits = vec![ + Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }, + Hit { doc_id: "eureka-state:mystery:X".into(), kind: "mystery".into(), score: 0.8 }, + ]; + + let mut among: Vec = entries.iter().map(|entry| entry.id.id.0.clone()).collect(); + among.push("eureka-state:mystery:X".into()); + let hits = projector.search("who owns the state", 3, &among, SEARCH_DEADLINE).unwrap(); + assert_eq!(hits.len(), 1, "the unknown kind is left out"); + assert_eq!(hits[0].0.id.0, "eureka-state:question:Q1"); + assert_eq!(hits[0].0.kind, PipelineKind::Question); + assert_eq!(hits[0].1, 0.9); + let embedded = embedder.texts(); + assert_eq!(embedded.last().unwrap(), &format!("Instruct: {QUERY_INSTRUCTION}\nQuery: who owns the state")); + projector.search("again", 100, &among, SEARCH_DEADLINE).unwrap(); + let state = index.state.lock().unwrap(); + assert_eq!(state.searches.iter().map(|(name, length, limit)| (name.as_str(), *length, *limit)).collect::>(), [ + (collection().as_str(), 4, 3), + (collection().as_str(), 4, 100), + ], "top_k as asked, with no oversample"); + let points: Vec = among.iter().map(|id| point_id(id)).collect(); + assert_eq!(state.among, [points.clone(), points], "the index was handed the members' point ids"); +} + +/// A search is never run against a collection that is not known to fit the +/// model: before reconciliation, and after the model has changed, it refuses +/// by saying so rather than comparing vectors of two models. +#[test] +fn a_search_refuses_when_the_index_is_not_reconciled_with_the_model() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + assert!(format!("{:#}", fresh.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()).contains("not been built")); + assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); + + let mut projector = projected(&embedder, &index, &entries); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + let before = embedder.texts().len(); + assert!(format!("{:#}", projector.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()).contains("model changed")); + assert_eq!(embedder.texts().len(), before, "the query was not embedded under the new model"); + assert!(index.state.lock().unwrap().searches.is_empty()); + assert!(projector.stale(), "and the reconciliation is to be redone"); +} + +/// The source delete names the repo, the paths when given, and, when it keeps a +/// commit, excludes that commit's points: the filter is what spares the chunks +/// just written. +#[test] +fn qdrant_source_delete_filters_by_repo_paths_and_the_commit_it_keeps() { + use crate::sources::SourceIndex; + let deleting = stub(&[("POST /collections/", 200, r#"{"result":{"status":"completed"},"status":"ok"}"#)]); + let mut store = QdrantIndex::new(&deleting.url); + store.delete_by("Demo", Some(&["a.rs".to_owned()]), Some("abc123")).unwrap(); + store.delete_by("Gone", None, None).unwrap(); + let seen = deleting.seen.lock().unwrap(); + let bodies: Vec = seen.iter().map(|(_, _, body)| serde_json::from_str(body).unwrap()).collect(); + assert!(seen.iter().all(|(method, path, _)| method == "POST" && path.ends_with("/points/delete?wait=true"))); + assert_eq!( + bodies[0]["filter"], + serde_json::json!({ + "must": [{ "key": "repo", "match": { "value": "Demo" } }, { "key": "path", "match": { "any": ["a.rs"] } }], + "must_not": [{ "key": "commit", "match": { "value": "abc123" } }] + }) + ); + assert_eq!(bodies[1]["filter"], serde_json::json!({ "must": [{ "key": "repo", "match": { "value": "Gone" } }] })); +} + +/// The search scores exactly the points it is handed (a has_id filter, exact +/// search; which documents are members is the mind's), and reads back the +/// document id, the kind and the score, in the order Qdrant gave them. A hit +/// missing what the join needs is an error, not a guess. +#[test] +fn qdrant_search_scores_only_the_members_exactly_and_reads_ids_kinds_and_scores() { + let name = collection(); + let answering = stub(&[( + "POST /collections/", + 200, + r#"{"result":[{"id":"p1","score":0.75,"payload":{"doc_id":"eureka-state:question:Q1","kind":"question"}},{"id":"p2","score":0.5,"payload":{"doc_id":"eureka-state:ruling:R1","kind":"ruling"}}]}"#, + )]); + let hits = QdrantIndex::new(&answering.url).search(&name, &[0.5, 1.0, 2.0, 3.0], &["p1".to_owned(), "p2".to_owned()], 12, TIME).unwrap(); + assert_eq!( + hits, + [ + Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.75 }, + Hit { doc_id: "eureka-state:ruling:R1".into(), kind: "ruling".into(), score: 0.5 }, + ] + ); + let seen = answering.seen.lock().unwrap(); + assert_eq!((seen[0].0.as_str(), seen[0].1.as_str()), ("POST", format!("/collections/{name}/points/search").as_str())); let body: serde_json::Value = serde_json::from_str(&seen[0].2).expect("the request body is JSON"); assert_eq!(body["vector"], serde_json::json!([0.5, 1.0, 2.0, 3.0])); assert_eq!(body["filter"], serde_json::json!({ "must": [{ "has_id": ["p1", "p2"] }] }), "members are included, by id alone"); @@ -1131,878 +1131,887 @@ fn qdrant_search_scores_only_the_members_exactly_and_reads_ids_kinds_and_scores( assert_eq!(body["limit"], 12); assert_eq!(body["with_vector"], false); assert!(body.get("exact").is_none(), "a top-level exact is ignored by Qdrant: {body}"); - drop(seen); - - // The limit is the caller's by value, across the range top_k takes: no - // constant, clamp or cap sits between the daemon and Qdrant. - for limit in [1, 12, 200] { - let echoing = stub(&[("POST /collections/", 200, r#"{"result":[]}"#)]); - QdrantIndex::new(&echoing.url).search(&name, &[1.0], &["p1".to_owned()], limit, TIME).unwrap(); - let sent: serde_json::Value = serde_json::from_str(&echoing.seen.lock().unwrap()[0].2).expect("the request body is JSON"); - assert_eq!(sent["limit"], limit, "the limit asked for is the limit sent"); - } - - let nameless = stub(&[("POST /collections/", 200, r#"{"result":[{"id":"p","score":0.5,"payload":{"kind":"question"}}]}"#)]); - assert!(QdrantIndex::new(&nameless.url).search(&name, &[1.0], &[], 1, TIME).is_err()); - let failing = stub(&[("POST /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); - assert!(QdrantIndex::new(&failing.url).search(&name, &[1.0], &[], 1, TIME).is_err()); -} - -fn queued(ticket: u64, text: &str) -> Queued { - Queued { ticket: SearchTicket(ticket), text: text.into(), top_k: 3, among: vec![], due: Instant::now() + SEARCH_DEADLINE } -} - -fn shared_with( - inbox: Vec, - mind: Option, - searches: Vec, - closed: bool, -) -> Lock { - Arc::new(( - Mutex::new(Shared { - inbox, - mind, - status: IndexStatus::Current, - closed, - searches: searches.into(), - found: Vec::new(), - source_searches: VecDeque::new(), - sources_found: Vec::new(), - next_ticket: 0, - }), - Condvar::new(), - )) -} - -/// The idle wait is for work: an entry, the mind's identity, a search or the -/// sink closing ends it at once, and only a quiet worker waits out its -/// deadline. The deadline is the caller's: a deadline already past ends the -/// wait at once too, and waking for work does not extend it. -#[test] -fn an_idle_wait_ends_for_work_and_otherwise_at_its_deadline() { - let (_root, mind) = mind_with_documents(); - let entry = mind.index_entries(None).unwrap().remove(0); - let waits = |shared: &Lock| { - let started = Instant::now(); - wait_for_work(shared, Wait::Work { until: Instant::now() + Duration::from_secs(30) }); - started.elapsed() < Duration::from_secs(10) - }; - assert!(waits(&shared_with(vec![entry], None, Vec::new(), false)), "an entry"); - assert!(waits(&shared_with(Vec::new(), Some(MIND.into()), Vec::new(), false)), "the mind's identity"); - assert!(waits(&shared_with(Vec::new(), None, vec![queued(0, "q")], false)), "a search"); - assert!(waits(&shared_with(Vec::new(), None, Vec::new(), true)), "the sink closing"); - - let quiet = shared_with(Vec::new(), None, Vec::new(), false); - let started = Instant::now(); - wait_for_work(&quiet, Wait::Work { until: started + Duration::from_millis(60) }); - assert!(started.elapsed() >= Duration::from_millis(60), "a quiet worker waits for its deadline"); - let started = Instant::now(); - wait_for_work(&quiet, Wait::Work { until: started.checked_sub(Duration::from_secs(1)).unwrap_or(started) }); - assert!(started.elapsed() < Duration::from_secs(10), "a deadline already past is not waited for"); -} - -/// A backoff is the whole delay, whatever arrives: a failing index is retried -/// on its schedule. -#[test] -fn a_backoff_wait_is_the_whole_delay_whatever_arrives() { - let (_root, mind) = mind_with_documents(); - let entry = mind.index_entries(None).unwrap().remove(0); - let busy = shared_with(vec![entry], Some(MIND.into()), vec![queued(0, "q")], false); - let started = Instant::now(); - wait_for_work(&busy, Wait::Backoff(Duration::from_millis(80))); - assert!(started.elapsed() >= Duration::from_millis(80), "the delay was waited out although work was queued"); - let closed = shared_with(Vec::new(), None, Vec::new(), true); - let started = Instant::now(); - wait_for_work(&closed, Wait::Backoff(Duration::from_secs(30))); - assert!(started.elapsed() < Duration::from_secs(10), "a closed sink ends even a backoff"); -} - -/// Searches accepted while the index was healthy are answered, with the -/// reason, when the worker fails: nothing is left to wait for its deadline. -#[test] -fn queued_searches_are_answered_with_why_when_the_worker_gives_up() { - let shared = shared_with(Vec::new(), None, vec![queued(3, "a"), queued(4, "b")], false); - refuse_searches(&shared, "the index could not advance: down"); - let guard = lock(&shared); - assert!(guard.searches.is_empty()); - assert_eq!( - guard.found, - [ - (SearchTicket(3), Err("the index could not advance: down".to_string())), - (SearchTicket(4), Err("the index could not advance: down".to_string())), - ] - ); -} - -// The semantic read's index side: when a search is answered, how many wait, -// and what the worker does between them. - -type Answers = Vec<(SearchTicket, Result)>; - -/// The sink's own calls, named: the trait is generic over the store and the -/// worker does not care which. -fn ask_sink(sink: &mut WorkerSink, text: &str, top_k: u32) -> Result { - >::search(sink, text, top_k, vec![]) -} - -fn collected(sink: &mut WorkerSink) -> Answers { - >::searched(sink) -} - -fn abandon(sink: &mut WorkerSink, ticket: SearchTicket) { - >::abandon(sink, ticket); -} - -/// Waits for `count` answers and returns them, in the order they were given. -fn answers(sink: &mut WorkerSink, count: usize) -> Answers { - let mut all = Vec::new(); - eventually("the searches being answered", || { - all.extend(collected(sink)); - all.len() >= count - }); - all -} - -/// `count` documents that share one document's text and differ in id, for the -/// tests whose subject is how many there are. -fn entries_of(count: usize) -> Vec { - let (_root, mind) = mind_with_documents(); - let template = mind.index_entries(None).unwrap().remove(0); - (0..count) - .map(|n| { - let mut entry = template.clone(); - entry.id.id.0 = format!("{}#{n}", entry.id.id.0); - entry - }) - .collect() -} - -fn queries(embedder: &FakeEmbedder) -> Vec { - embedder.texts().into_iter().filter(|text| text.starts_with("Instruct: ")).collect() -} - -/// A search is refused only while the index is being rebuilt or cannot be -/// asked, and the words name the state and how many documents are pending. Lag -/// is not a rebuild: the sink and the projector ask the one predicate and both -/// let it through. -#[test] -fn a_search_is_refused_only_for_a_rebuild_or_a_failure() { - use IndexStatus as S; - assert_eq!(unsearchable(&S::Current), None); - assert_eq!(unsearchable(&S::Behind { pending: 5 }), None, "lag is waited out, not refused"); - let says = |status: S, wanted: &[&str]| { - let why = unsearchable(&status).unwrap_or_else(|| panic!("{status:?} answered")); - for word in wanted { - assert!(why.contains(word), "{status:?}: {word} in {why}"); - } - }; - says(S::Reconciling { pending: 7 }, &["being rebuilt", "reconciling", "pending: 7"]); - says(S::Failing { pending: 3, attempts: 2, error: "boom".into() }, &["failing", "boom", "pending: 3"]); - says(S::Refused { pending: 4, attempts: 2, reason: "not ours".into() }, &["refused", "not ours", "pending: 4"]); - - // The sink: what the worker last published, and the documents admitted - // since, which are lag whatever the published state says of a current index. - let sink_in = |status: S, inbox: usize| { - let shared = shared_with(entries_of(inbox), None, Vec::new(), false); - lock(&shared).status = status; - WorkerSink { shared, identified: true, deadline: SEARCH_DEADLINE } - }; - assert!(ask_sink(&mut sink_in(S::Current, 2), "anything", 3).is_ok(), "admitted a moment ago and not yet absorbed"); - assert!(ask_sink(&mut sink_in(S::Behind { pending: 4 }, 0), "anything", 3).is_ok(), "the worker is writing"); - let refusal = ask_sink(&mut sink_in(S::Reconciling { pending: 4 }, 2), "anything", 3).unwrap_err(); - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 6"), "{refusal}"); - let refusal = ask_sink(&mut sink_in(S::Failing { pending: 1, attempts: 1, error: "boom".into() }, 0), "anything", 3).unwrap_err(); - assert!(refusal.contains("failing") && refusal.contains("boom"), "{refusal}"); - - // The projector: a rebuild refuses without touching the ports to find out. - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let (lost_embedder, lost_index) = pair(); - let mut rebuilding_now = rebuilding(&lost_embedder, &lost_index, &entries); - let searches = lost_index.state.lock().unwrap().searches.len(); - let refusal = format!("{:#}", rebuilding_now.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()); - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); - assert_eq!(lost_index.state.lock().unwrap().searches.len(), searches, "the rebuilt collection was not asked"); - assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); - - // Lag it lets through: catching up is the worker's, before the search is taken. - let mut lagging = projected(&embedder, &index, &entries[..3]); - lagging.want(entries[3..].iter().cloned()); - assert_eq!(lagging.progress_status(), IndexStatus::Behind { pending: 1 }); - assert!(lagging.search("anything", 3, &[], SEARCH_DEADLINE).is_ok()); -} - -/// A search that fails for its own reasons (the vector store errs, the -/// embedder is down, an identity read times out) fails that search alone: the -/// reconciliation stands, the status stays `Current`, the collection is not -/// listed again, and the next search is asked and answered. -#[test] -fn a_failed_search_fails_that_search_alone() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - type Fault = Box; - let faults: [(&str, Fault); 3] = [ - ("the vector store fails the search", Box::new(|_, index, on| index.state.lock().unwrap().fail_search = on)), - ("the embedder is down", Box::new(|embedder, _, on| embedder.state.lock().unwrap().down = on)), - ("the identity read never answers", Box::new(|embedder, _, on| embedder.state.lock().unwrap().hang_identity = on)), - ]; - for (what, fault) in faults { - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - let listed = index.state.lock().unwrap().listed; - fault(&embedder, &index, true); - assert!(projector.search("anything", 3, &[], Duration::from_millis(50)).is_err(), "{what}"); - assert_eq!(projector.progress_status(), IndexStatus::Current, "{what}"); - assert!(!projector.stale(), "{what}"); - fault(&embedder, &index, false); - assert!(projector.search("anything", 3, &[], SEARCH_DEADLINE).is_ok(), "{what}: the next search is answered"); - assert_eq!(index.state.lock().unwrap().listed, listed, "{what}: the collection was not listed again"); - } - - // In the worker the failed search is answered and nothing else moves. - let (embedder, index) = pair(); - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let listed = index.state.lock().unwrap().listed; - index.state.lock().unwrap().fail_search = true; - let ticket = ask_sink(&mut sink, "anything", 3).unwrap(); - let failed = answers(&mut sink, 1); - assert_eq!(failed[0].0, ticket); - assert!(failed[0].1.is_err()); - index.state.lock().unwrap().fail_search = false; - let next = ask_sink(&mut sink, "anything", 3).expect("the index is still current, not being rebuilt"); - let answered = answers(&mut sink, 1); - assert_eq!((answered[0].0, answered[0].1.is_ok()), (next, true)); - assert_eq!(index.state.lock().unwrap().listed, listed, "the worker did not list the collection again"); -} - -/// A model the search finds changed is a rebuild, and its refusal names the -/// state and how many documents are pending, as every other refusal does. -#[test] -fn a_model_changed_refusal_names_the_state_and_the_pending_count() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - projector.want(entries[3..].iter().cloned()); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - let refusal = format!("{:#}", projector.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()); - assert!(refusal.contains("model changed") && refusal.contains("being rebuilt") && refusal.contains("pending: 1"), "{refusal}"); -} - -/// A search asks the store what the collection says about itself: a label for -/// another mind, no label, or no collection is refused before the query is -/// embedded, and the reconciliation is to be redone. -#[test] -fn a_search_re_reads_the_collections_label() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let cases: [Edit; 3] = [ - ( - "another mind's label", - Box::new(|stored| { - let Described::Labelled(meta) = &mut stored.label else { panic!() }; - meta.mind = "mind-commit-another".into(); - }), - ), - ("no label", Box::new(|stored| stored.label = Described::Unlabelled)), - ("another text version", Box::new(|stored| { - let Described::Labelled(meta) = &mut stored.label else { panic!() }; - meta.index_text_version += 1; - })), - ]; - for (what, change) in cases { - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - change(index.state.lock().unwrap().collections.get_mut(&collection()).unwrap()); - let embedded = embedder.texts().len(); - let refusal = format!("{:#}", projector.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()); - assert!(refusal.contains("no longer the one this mind's index was built as"), "{what}: {refusal}"); - assert_eq!(embedder.texts().len(), embedded, "{what}: nothing was embedded"); - assert!(index.state.lock().unwrap().searches.is_empty(), "{what}"); - assert!(projector.stale(), "{what}"); - } - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - index.state.lock().unwrap().collections.clear(); - assert!(projector.search("anything", 3, &[], SEARCH_DEADLINE).is_err(), "no collection"); - assert!(projector.stale()); -} - -/// The model is read again after the query is embedded: a vector made while the -/// model changed is never compared with the collection. -#[test] -fn a_model_that_changes_while_the_query_is_embedded_is_refused() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let gate = Gate::shut(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let index = FakeIndex::default(); - gate.open(); - let mut projector = projected(&embedder, &index, &entries); - gate.shut_again(); - let searching = std::thread::spawn(move || { - let found = projector.search("anything", 3, &[], SEARCH_DEADLINE); - (projector, found) - }); - gate.wait_arrived(1); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - gate.open(); - let (projector, found) = searching.join().unwrap(); - let refusal = format!("{:#}", found.unwrap_err()); - assert!(refusal.contains("while the texts were embedded"), "{refusal}"); - assert!(index.state.lock().unwrap().searches.is_empty(), "the vector was not used"); - assert!(projector.stale()); -} - -/// Each search has a ticket of its own, and its answer comes back once, as the -/// worker gave it: nothing is added to it and nothing is left to be -/// collected twice. -#[test] -fn every_search_has_its_own_ticket_and_its_answer_comes_back_exactly_once() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - index.state.lock().unwrap().hits = - vec![Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }]; - let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let expected: Hits = vec![( - PipelineRef { kind: PipelineKind::Question, id: Short("eureka-state:question:Q1".into()) }, - 0.9, - )]; - let mut tickets = Vec::new(); - for text in ["a", "b", "c"] { - let among = vec!["eureka-state:question:Q1".to_owned()]; - let ticket = >::search(&mut sink, text, 3, among).unwrap(); - assert_eq!(answers(&mut sink, 1), vec![(ticket, Ok(expected.clone()))]); - assert!(collected(&mut sink).is_empty(), "an answer is handed over once"); - tickets.push(ticket); - } - assert_eq!(tickets, [SearchTicket(0), SearchTicket(1), SearchTicket(2)]); -} - -/// The queue of searches is bounded and the asker who does not fit is told. -/// One search is inside the embedder and the queue is full; the next is -/// refused by name, and when the worker has answered some it takes more. -#[test] -fn the_search_queue_is_bounded_and_the_asker_is_told_when_it_is_full() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - ask_sink(&mut sink, "in flight", 3).unwrap(); - gate.wait_arrived(1); - for n in 0..SEARCHES_QUEUED_MAX { - ask_sink(&mut sink, &format!("waiting {n}"), 3).unwrap_or_else(|why| panic!("{n} of {SEARCHES_QUEUED_MAX}: {why}")); - } - let refusal = ask_sink(&mut sink, "one too many", 3).unwrap_err(); - assert!(refusal.contains(&format!("{SEARCHES_QUEUED_MAX} searches waiting")), "{refusal}"); - - gate.open(); - let answered = answers(&mut sink, SEARCHES_QUEUED_MAX + 1); - assert_eq!(answered.len(), SEARCHES_QUEUED_MAX + 1); - assert!(ask_sink(&mut sink, "room again", 3).is_ok()); -} - -/// A search the asker has stopped waiting for is taken out of the queue and -/// never embedded; the ones on either side of it are. -#[test] -fn an_abandoned_search_is_never_embedded() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let first = ask_sink(&mut sink, "first", 3).unwrap(); - gate.wait_arrived(1); - let second = ask_sink(&mut sink, "second", 3).unwrap(); - let third = ask_sink(&mut sink, "third", 3).unwrap(); - abandon(&mut sink, second); - gate.open(); - let answered = answers(&mut sink, 2); - let tickets: Vec = answered.iter().map(|(ticket, _)| *ticket).collect(); - assert_eq!(tickets, [first, third]); - let texts = queries(&embedder); - assert_eq!(texts.len(), 2, "{texts:?}"); - assert!(texts[0].ends_with("Query: first") && texts[1].ends_with("Query: third"), "{texts:?}"); -} - -/// The worker serves one queued search and returns to its writes: three -/// waiting searches leave two after one turn. -#[test] -fn one_queued_search_is_served_per_turn() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - let shared = shared_with(Vec::new(), None, vec![queued(1, "a"), queued(2, "b"), queued(3, "c")], false); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert_eq!(guard.searches.iter().map(|queued| queued.ticket).collect::>(), [SearchTicket(2), SearchTicket(3)]); - assert_eq!(guard.found.iter().map(|(ticket, _)| *ticket).collect::>(), [SearchTicket(1)]); -} - -/// A search reads its own writes. Asked while a batch is in the embedder and -/// eight more documents wait, it is accepted, waits behind them, and is -/// answered only once every one is in the collection. -#[test] -fn a_search_waits_for_the_writes_ahead_of_it_and_reads_them() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), quick()); - gate.wait_arrived(1); - let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); - gate.open(); - let answered = answers(&mut sink, 1); - assert_eq!(answered[0].0, ticket); - assert!(answered[0].1.is_ok(), "{answered:?}"); - assert_eq!(index.state.lock().unwrap().held_when_searched, [BATCH + 8], "the search saw every document admitted before it"); - sink.wait_status(|status| *status == IndexStatus::Current); -} - -/// The worker takes admissions before the search, and leaves the search queued -/// while the writes ahead of it are unfinished: a search asked with a document -/// still in the inbox absorbs it and waits, and once the index has caught up -/// it is answered. An index being rebuilt is not waited on: the search is taken -/// and refused, naming the state and the pending count. -#[test] -fn the_worker_absorbs_admissions_before_a_search_and_waits_for_lag_but_not_for_a_rebuild() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - let shared = shared_with(vec![entries[3].clone()], None, vec![queued(1, "a")], false); - - serve_search(&mut projector, &shared); - { - let guard = lock(&shared); - assert!(guard.inbox.is_empty(), "the admission ahead of the search was absorbed"); - assert_eq!(guard.searches.len(), 1, "the search waits for it to be written"); - assert!(guard.found.is_empty()); - } - assert_eq!(projector.pending(), 1); - - drain(&mut projector); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert!(guard.searches.is_empty()); - assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); - assert_eq!(index.state.lock().unwrap().held_when_searched, [4]); - drop(guard); - - // A rebuild: the collection was lost and made again, and every entry is owed. - let (rebuild_embedder, rebuild_index) = pair(); - let mut rebuilding = rebuilding(&rebuild_embedder, &rebuild_index, &entries); - let shared = shared_with(Vec::new(), None, vec![queued(2, "b")], false); - serve_search(&mut rebuilding, &shared); - let guard = lock(&shared); - let [(SearchTicket(2), Err(refusal))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); -} - -/// A collection made again with nothing to refill owes nothing, so the rebuild -/// ends inside `reconcile`: the next entry is lag, and a search waits for it -/// rather than being refused as a rebuild. -#[test] -fn a_rebuild_that_owes_nothing_settles_when_it_is_made() { - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into())); - drain(&mut projector); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - assert_eq!(projector.recheck().unwrap(), Advance::Progressed, "the collection is made again under the new digest"); - assert_eq!(projector.pending(), 0); - - projector.want(entries_of(1)); - assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 1 }, "nothing was owed, so nothing is being rebuilt"); - let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert_eq!(guard.searches.len(), 1, "the search waits for the write"); - assert!(guard.found.is_empty(), "and is not refused: {:?}", guard.found); -} - -/// An identified mind with nothing owed has still to reconcile once: a search -/// waits for that, and is not taken to fail against an index not yet built. -#[test] -fn a_search_waits_for_the_first_reconcile_of_an_identified_mind_owing_nothing() { - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); - assert_eq!(projector.pending(), 0); - assert!(projector.stale()); - let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); - serve_search(&mut projector, &shared); - { - let guard = lock(&shared); - assert_eq!(guard.searches.len(), 1, "the search waits for the reconcile"); - assert!(guard.found.is_empty(), "{:?}", guard.found); - } - drain(&mut projector); - serve_search(&mut projector, &shared); - assert!(matches!(lock(&shared).found.as_slice(), [(SearchTicket(1), Ok(_))])); -} - -/// A search that was queued when the worker failed is answered with why, by -/// the worker's own failure path. -#[test] -fn a_search_queued_before_the_worker_fails_is_answered_with_why() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries_of(1), quick()); - gate.wait_arrived(1); - lock(&sink.shared).searches.push_back(queued(5, "anything")); - embedder.state.lock().unwrap().down = true; - gate.open(); - let answered = answers(&mut sink, 1); - assert_eq!(answered[0].0, SearchTicket(5)); - let Err(why) = &answered[0].1 else { panic!("{answered:?}") }; - assert!(why.contains("the semantic index is failing") && why.contains("pending: 1") && why.contains("the embedder is down"), "{why}"); -} - -/// While a failure is being reported, the report's pending count is the one -/// the worker holds now, published under the lock that took the entries out of -/// the inbox: never the count from before they moved. -#[test] -fn a_failure_report_carries_the_pending_count_it_has_now() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); - projector.want(entries[..2].iter().cloned()); - let shared = shared_with(entries[2..].to_vec(), None, Vec::new(), false); - let error = StepError::Unavailable(anyhow::anyhow!("down")); - assert!(absorb(&mut projector, &shared, Some((&error, 3)))); - { - let guard = lock(&shared); - assert!(guard.inbox.is_empty()); - assert_eq!(guard.status, IndexStatus::Failing { pending: 4, attempts: 3, error: "down".into() }); - } - assert!(absorb(&mut projector, &shared, None)); - assert_eq!(lock(&shared).status, IndexStatus::Behind { pending: 4 }, "a first build is lag"); - lock(&shared).closed = true; - assert!(!absorb(&mut projector, &shared, None), "a closed sink ends the worker"); -} - -/// The worker's recheck is on its own clock: searches arriving faster than the -/// interval do not keep pushing it out. -#[test] -fn steady_searches_do_not_postpone_the_workers_recheck() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let backoff = Backoff { recheck: Duration::from_millis(200), ..quick() }; - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); - sink.wait_status(|status| *status == IndexStatus::Current); - let listed = index.state.lock().unwrap().listed; - let started = Instant::now(); - while index.state.lock().unwrap().listed < listed + 2 { - assert!(started.elapsed() < Duration::from_secs(5), "the worker never rechecked while searches kept arriving"); - let _ = ask_sink(&mut sink, "anything", 3); - collected(&mut sink); - std::thread::sleep(Duration::from_millis(10)); - } -} - -/// `wait_status` gives up at one deadline, not one per wake: a health that -/// keeps changing does not keep it waiting. -#[test] -fn wait_status_gives_up_at_its_overall_deadline_however_often_the_health_changes() { - let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let shared = Arc::clone(&sink.shared); - let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let ticking = Arc::clone(&stop); - let ticker = std::thread::spawn(move || { - while !ticking.load(std::sync::atomic::Ordering::Relaxed) { - shared.1.notify_all(); - std::thread::sleep(Duration::from_millis(5)); - } - }); - let (tell, told) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let gave_up = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - sink.wait_status_for(Duration::from_millis(300), |status| *status == IndexStatus::Reconciling { pending: 99 }) - })) - .is_err(); - let _ = tell.send(gave_up); - }); - let gave_up = told.recv_timeout(Duration::from_secs(10)); - stop.store(true, std::sync::atomic::Ordering::Relaxed); - ticker.join().unwrap(); - assert_eq!(gave_up, Ok(true), "the wait did not end at its deadline"); -} - -/// An idle worker verifies once per interval, not in a loop: the recheck moves -/// its own deadline when it runs. -#[test] -fn an_idle_worker_rechecks_once_per_interval() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let backoff = Backoff { recheck: Duration::from_millis(100), ..quick() }; - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); - sink.wait_status(|status| *status == IndexStatus::Current); - std::thread::sleep(Duration::from_millis(600)); - let listed = index.state.lock().unwrap().listed; - assert!((2..=40).contains(&listed), "the startup reconciliation and about five rechecks listed the collection {listed} times"); -} - -// The search's rulings: what may be waiting ahead of it, when a rebuild -// refuses, what a first build is, how time is spent, and who owns the deadline. - -/// A search is never taken while a document sits in the inbox. An admission -/// that lands after the inbox was absorbed and before the search was taken (the -/// hook puts one there, where a second lock once left the gap) leaves the -/// search queued; the next turn absorbs it, and the search is answered only -/// once it is written. -#[test] -fn a_search_is_never_taken_while_an_admission_sits_in_the_inbox() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); - let late = entries[3].clone(); - AFTER_ABSORB.with(|hook| *hook.borrow_mut() = Some(Box::new(move |guard| guard.inbox.push(late)))); - - serve_search(&mut projector, &shared); - { - let guard = lock(&shared); - assert_eq!(guard.searches.len(), 1, "the search was not taken past the admission"); - assert!(guard.found.is_empty()); - assert_eq!(guard.inbox.len(), 1); - } - serve_search(&mut projector, &shared); - assert_eq!(lock(&shared).searches.len(), 1, "absorbed, but not yet written"); - drain(&mut projector); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); - assert_eq!(index.state.lock().unwrap().held_when_searched, [4], "the search saw the admission"); -} - -/// Through the sink, an admission acknowledged before the search was asked is -/// in the inbox while the worker is inside a batch; the search reads it. -#[test] -fn a_search_reads_an_admission_acknowledged_before_it_was_asked() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(2), quick()); - gate.wait_arrived(1); - let admitted = entries_of(5).split_off(2); - lock(&sink.shared).inbox.extend(admitted); - let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); - gate.open(); - let answered = answers(&mut sink, 1); - assert_eq!(answered[0].0, ticket); - assert!(answered[0].1.is_ok(), "{answered:?}"); - assert_eq!(index.state.lock().unwrap().held_when_searched, [5]); -} - -/// A collection made again is a rebuild until every document it owes is -/// written again: searches are refused as rebuilding, however the collection -/// came to be made again, and however many steps or failures the refill takes. -#[test] -fn a_collection_made_again_refuses_searches_until_it_is_refilled() { - let refused = |projector: &mut Projector, pending: u32| { - let why = format!("{:#}", projector.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()); - assert!(why.contains("being rebuilt") && why.contains(&format!("pending: {pending}")), "{why}"); - }; - let entries = entries_of(BATCH + 8); - - // Lost while the worker ran. - let (embedder, index) = pair(); - let mut projector = rebuilding(&embedder, &index, &entries); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); - refused(&mut projector, 40); - index.state.lock().unwrap().down = true; - projector.advance().expect_err("the store is down mid-refill"); - index.state.lock().unwrap().down = false; - assert_eq!(projector.advance().unwrap(), Advance::Progressed, "reconciled again, under a label that already fits"); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }, "a failure does not end the rebuild"); - assert_eq!(projector.advance().unwrap(), Advance::Progressed); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }, "half refilled is still a rebuild"); - refused(&mut projector, 8); - assert_eq!(projector.advance().unwrap(), Advance::Progressed); - assert_eq!(projector.progress_status(), IndexStatus::Current); - assert!(projector.search("anything", 3, &[], SEARCH_DEADLINE).is_ok(), "answered once refilled"); - assert_eq!(index.state.lock().unwrap().held_when_searched, [40]); - projector.want(entries_of(1).into_iter().map(|mut entry| { - entry.id.id.0.push_str("-later"); - entry - })); - assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 1 }, "a later admission is lag, not a rebuild"); - - // Made again because the model changed. - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - assert_eq!(projector.recheck().unwrap(), Advance::Progressed, "the change is seen and the collection made again"); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); - projector.advance().unwrap(); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }); - refused(&mut projector, 8); - drain(&mut projector); - assert!(projector.search("anything", 3, &[], SEARCH_DEADLINE).is_ok()); -} - -/// Through the worker: a collection lost while the worker runs is made again, -/// and a search asked during the refill is refused as rebuilding, naming how -/// many documents are pending; one asked after it is answered. -#[test] -fn a_search_asked_during_a_refill_is_refused_and_one_after_it_is_answered() { - let gate = Gate::shut(); - gate.open(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let backoff = Backoff { recheck: Duration::from_millis(50), ..quick() }; - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), backoff); - sink.wait_status(|status| *status == IndexStatus::Current); - gate.shut_again(); - index.state.lock().unwrap().collections.remove(&collection()); - gate.wait_arrived(1); - let refusal = ask_sink(&mut sink, "anything", 3).unwrap_err(); - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 40"), "{refusal}"); - gate.open(); - sink.wait_status(|status| *status == IndexStatus::Current); - let ticket = ask_sink(&mut sink, "anything", 3).expect("refilled"); - let answered = answers(&mut sink, 1); - assert_eq!((answered[0].0, answered[0].1.is_ok()), (ticket, true)); - assert_eq!(index.state.lock().unwrap().held_when_searched.last(), Some(&40)); -} - -/// A mind whose index has never been built has nothing stale to protect: its -/// first build is lag, whether the search was asked before the worker had -/// looked at the first admission or after. Only a rebuild of an index that -/// exists refuses. -#[test] -fn a_minds_first_build_is_lag_whoever_looks_first() { - for asked_before_the_worker_looked in [true, false] { - let what = if asked_before_the_worker_looked { "asked first" } else { "asked after the worker looked" }; - let (embedder, index) = pair(); - let entries = entries_of(3); - let (mut projector, shared) = if asked_before_the_worker_looked { - let projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); - (projector, shared_with(entries, Some(MIND.into()), Vec::new(), false)) - } else { - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries); - let shared = shared_with(Vec::new(), None, Vec::new(), false); - lock(&shared).status = projector.progress_status(); - (projector, shared) - }; - let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline: SEARCH_DEADLINE }; - let ticket = ask_sink(&mut sink, "anything", 3).unwrap_or_else(|why| panic!("{what}: {why}")); - - serve_search(&mut projector, &shared); - assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 3 }, "{what}: not yet reconciled, and lag"); - assert_eq!(lock(&shared).searches.len(), 1, "{what}: the search waits for the first reconciliation"); - assert_eq!(projector.advance().unwrap(), Advance::Progressed); - serve_search(&mut projector, &shared); - assert_eq!(lock(&shared).searches.len(), 1, "{what}: and for the first writes"); - drain(&mut projector); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert!(matches!(guard.found.as_slice(), [(found, Ok(_))] if *found == ticket), "{what}: {:?}", guard.found); - assert_eq!(index.state.lock().unwrap().held_when_searched, [3], "{what}"); - } - - let entries = entries_of(4); - let (embedder, index) = pair(); - projected(&embedder, &index, &entries[..3]); - let mut partial = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - partial.want(entries.iter().cloned()); - partial.advance().unwrap(); - assert_eq!(partial.progress_status(), IndexStatus::Behind { pending: 1 }, "an index found whole at start-up only lags"); - drain(&mut partial); - index.state.lock().unwrap().collections.remove(&collection()); - assert_eq!(partial.recheck().unwrap(), Advance::Progressed); - assert_eq!(partial.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index found at start-up was built: losing it is a rebuild"); - - embedder.state.lock().unwrap().identity.digest = "d2".into(); - let mut replaced = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); - replaced.want(entries); - replaced.advance().unwrap(); - assert_eq!(replaced.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index that exists and is made again is a rebuild"); -} - -/// A mind with no identity and documents owed has nothing to wait for: the -/// search is taken and refused, not left queued for a worker that would spin -/// on it. -#[test] -fn a_search_is_refused_when_documents_are_owed_to_a_mind_with_no_identity() { - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); - projector.want(entries_of(2)); - let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert!(guard.searches.is_empty()); - let [(SearchTicket(1), Err(why))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; - assert!(why.contains("being rebuilt") && why.contains("pending: 2"), "{why}"); - assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); -} - -/// Every call a search makes is bounded by what its time has left, not by the -/// time it was given: after a slow identity read the embed's bound is smaller -/// by what the read took, and the query's by both reads. -#[test] -fn a_later_calls_bound_shrinks_as_the_search_spends_its_time() { - let entries = entries_of(3); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - let delay = Duration::from_millis(150); - let within = Duration::from_secs(10); - { - let mut state = embedder.state.lock().unwrap(); - state.identity_delay = delay; - state.bounds.clear(); - state.identity_bounds.clear(); - } - index.state.lock().unwrap().bounds.clear(); - projector.search("anything", 3, &[], within).unwrap(); - - let embedder = embedder.state.lock().unwrap(); - let index = index.state.lock().unwrap(); - assert!(embedder.identity_bounds[0] <= within); - assert!(embedder.bounds[0] <= within - delay, "the embed came after one slow read: {:?}", embedder.bounds); - assert!(embedder.identity_bounds[1] <= within - delay, "{:?}", embedder.identity_bounds); - let queried = index.bounds.iter().find(|(call, _)| *call == "search").unwrap().1; - assert!(queried <= within - delay * 2, "the query came after two slow reads: {queried:?}"); -} - -/// The deadline has one owner: the sink. It stamps every search with it, and -/// it is what the serve loop is told (`Daemon::search_deadline`), so the loop's -/// wait and the worker's bound are one value. -#[test] -fn the_sink_owns_the_searchs_deadline_and_tells_the_loop() { - let (_root, mind) = mind_with_documents(); - let deadline = Duration::from_secs(7); - let shared = shared_with(Vec::new(), None, Vec::new(), false); - let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline }; - let asked = Instant::now(); - ask_sink(&mut sink, "anything", 3).unwrap(); - let due = lock(&shared).searches.back().expect("no worker took it").due; - assert!(due >= asked + deadline && due <= Instant::now() + deadline, "stamped with the sink's deadline"); - assert_eq!(>::search_deadline(&sink), deadline); - assert_eq!(Daemon::new(mind, sink).search_deadline(), deadline); -} + drop(seen); + + // The limit is the caller's by value, across the range top_k takes: no + // constant, clamp or cap sits between the daemon and Qdrant. + for limit in [1, 12, 200] { + let echoing = stub(&[("POST /collections/", 200, r#"{"result":[]}"#)]); + QdrantIndex::new(&echoing.url).search(&name, &[1.0], &["p1".to_owned()], limit, TIME).unwrap(); + let sent: serde_json::Value = serde_json::from_str(&echoing.seen.lock().unwrap()[0].2).expect("the request body is JSON"); + assert_eq!(sent["limit"], limit, "the limit asked for is the limit sent"); + } + + let nameless = stub(&[("POST /collections/", 200, r#"{"result":[{"id":"p","score":0.5,"payload":{"kind":"question"}}]}"#)]); + assert!(QdrantIndex::new(&nameless.url).search(&name, &[1.0], &[], 1, TIME).is_err()); + let failing = stub(&[("POST /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); + assert!(QdrantIndex::new(&failing.url).search(&name, &[1.0], &[], 1, TIME).is_err()); +} + +fn queued(ticket: u64, text: &str) -> Queued { + Queued { ticket: SearchTicket(ticket), text: text.into(), top_k: 3, among: vec![], due: Instant::now() + SEARCH_DEADLINE } +} + +fn shared_with( + inbox: Vec, + mind: Option, + searches: Vec, + closed: bool, +) -> Lock { + Arc::new(( + Mutex::new(Shared { + inbox, + mind, + status: IndexStatus::Current, + closed, + searches: searches.into(), + found: Vec::new(), + source_searches: VecDeque::new(), + sources_found: Vec::new(), + next_ticket: 0, + }), + Condvar::new(), + )) +} + +/// The idle wait is for work: an entry, the mind's identity, a search or the +/// sink closing ends it at once, and only a quiet worker waits out its +/// deadline. The deadline is the caller's: a deadline already past ends the +/// wait at once too, and waking for work does not extend it. +#[test] +fn an_idle_wait_ends_for_work_and_otherwise_at_its_deadline() { + let (_root, mind) = mind_with_documents(); + let entry = mind.index_entries(None).unwrap().remove(0); + let waits = |shared: &Lock| { + let started = Instant::now(); + wait_for_work(shared, Wait::Work { until: Instant::now() + Duration::from_secs(30) }); + started.elapsed() < Duration::from_secs(10) + }; + assert!(waits(&shared_with(vec![entry], None, Vec::new(), false)), "an entry"); + assert!(waits(&shared_with(Vec::new(), Some(MIND.into()), Vec::new(), false)), "the mind's identity"); + assert!(waits(&shared_with(Vec::new(), None, vec![queued(0, "q")], false)), "a search"); + assert!(waits(&shared_with(Vec::new(), None, Vec::new(), true)), "the sink closing"); + + let quiet = shared_with(Vec::new(), None, Vec::new(), false); + let started = Instant::now(); + wait_for_work(&quiet, Wait::Work { until: started + Duration::from_millis(60) }); + assert!(started.elapsed() >= Duration::from_millis(60), "a quiet worker waits for its deadline"); + let started = Instant::now(); + wait_for_work(&quiet, Wait::Work { until: started.checked_sub(Duration::from_secs(1)).unwrap_or(started) }); + assert!(started.elapsed() < Duration::from_secs(10), "a deadline already past is not waited for"); +} + +/// A backoff is the whole delay, whatever arrives: a failing index is retried +/// on its schedule. +#[test] +fn a_backoff_wait_is_the_whole_delay_whatever_arrives() { + let (_root, mind) = mind_with_documents(); + let entry = mind.index_entries(None).unwrap().remove(0); + let busy = shared_with(vec![entry], Some(MIND.into()), vec![queued(0, "q")], false); + let started = Instant::now(); + wait_for_work(&busy, Wait::Backoff(Duration::from_millis(80))); + assert!(started.elapsed() >= Duration::from_millis(80), "the delay was waited out although work was queued"); + let closed = shared_with(Vec::new(), None, Vec::new(), true); + let started = Instant::now(); + wait_for_work(&closed, Wait::Backoff(Duration::from_secs(30))); + assert!(started.elapsed() < Duration::from_secs(10), "a closed sink ends even a backoff"); +} + +/// Searches accepted while the index was healthy are answered, with the +/// reason, when the worker fails: nothing is left to wait for its deadline. +#[test] +fn queued_searches_are_answered_with_why_when_the_worker_gives_up() { + let shared = shared_with(Vec::new(), None, vec![queued(3, "a"), queued(4, "b")], false); + refuse_searches(&shared, "the index could not advance: down"); + let guard = lock(&shared); + assert!(guard.searches.is_empty()); + assert_eq!( + guard.found, + [ + (SearchTicket(3), Err("the index could not advance: down".to_string())), + (SearchTicket(4), Err("the index could not advance: down".to_string())), + ] + ); +} + +// The semantic read's index side: when a search is answered, how many wait, +// and what the worker does between them. + +type Answers = Vec<(SearchTicket, Result)>; + +/// The sink's own calls, named: the trait is generic over the store and the +/// worker does not care which. +fn ask_sink(sink: &mut WorkerSink, text: &str, top_k: u32) -> Result { + >::search(sink, text, top_k, vec![]) +} + +fn collected(sink: &mut WorkerSink) -> Answers { + >::searched(sink) +} + +fn abandon(sink: &mut WorkerSink, ticket: SearchTicket) { + >::abandon(sink, ticket); +} + +/// Waits for `count` answers and returns them, in the order they were given. +fn answers(sink: &mut WorkerSink, count: usize) -> Answers { + let mut all = Vec::new(); + eventually("the searches being answered", || { + all.extend(collected(sink)); + all.len() >= count + }); + all +} + +/// `count` documents that share one document's text and differ in id, for the +/// tests whose subject is how many there are. +fn entries_of(count: usize) -> Vec { + let (_root, mind) = mind_with_documents(); + let template = mind.index_entries(None).unwrap().remove(0); + (0..count) + .map(|n| { + let mut entry = template.clone(); + entry.id.id.0 = format!("{}#{n}", entry.id.id.0); + entry + }) + .collect() +} + +fn queries(embedder: &FakeEmbedder) -> Vec { + embedder.texts().into_iter().filter(|text| text.starts_with("Instruct: ")).collect() +} + +/// A search is refused only while the index is being rebuilt or cannot be +/// asked, and the words name the state and how many documents are pending. Lag +/// is not a rebuild: the sink and the projector ask the one predicate and both +/// let it through. +#[test] +fn a_search_is_refused_only_for_a_rebuild_or_a_failure() { + use IndexStatus as S; + assert_eq!(unsearchable(&S::Current), None); + assert_eq!(unsearchable(&S::Behind { pending: 5 }), None, "lag is waited out, not refused"); + let says = |status: S, wanted: &[&str]| { + let why = unsearchable(&status).unwrap_or_else(|| panic!("{status:?} answered")); + for word in wanted { + assert!(why.contains(word), "{status:?}: {word} in {why}"); + } + }; + says(S::Reconciling { pending: 7 }, &["being rebuilt", "reconciling", "pending: 7"]); + says(S::Failing { pending: 3, attempts: 2, error: "boom".into() }, &["failing", "boom", "pending: 3"]); + says(S::Refused { pending: 4, attempts: 2, reason: "not ours".into() }, &["refused", "not ours", "pending: 4"]); + + // The sink: what the worker last published, and the documents admitted + // since, which are lag whatever the published state says of a current index. + let sink_in = |status: S, inbox: usize| { + let shared = shared_with(entries_of(inbox), None, Vec::new(), false); + lock(&shared).status = status; + WorkerSink { shared, identified: true, deadline: SEARCH_DEADLINE } + }; + assert!(ask_sink(&mut sink_in(S::Current, 2), "anything", 3).is_ok(), "admitted a moment ago and not yet absorbed"); + assert!(ask_sink(&mut sink_in(S::Behind { pending: 4 }, 0), "anything", 3).is_ok(), "the worker is writing"); + let refusal = ask_sink(&mut sink_in(S::Reconciling { pending: 4 }, 2), "anything", 3).unwrap_err(); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 6"), "{refusal}"); + let refusal = ask_sink(&mut sink_in(S::Failing { pending: 1, attempts: 1, error: "boom".into() }, 0), "anything", 3).unwrap_err(); + assert!(refusal.contains("failing") && refusal.contains("boom"), "{refusal}"); + + // The projector: a rebuild refuses without touching the ports to find out. + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let (lost_embedder, lost_index) = pair(); + let mut rebuilding_now = rebuilding(&lost_embedder, &lost_index, &entries); + let searches = lost_index.state.lock().unwrap().searches.len(); + let refusal = format!("{:#}", rebuilding_now.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); + assert_eq!(lost_index.state.lock().unwrap().searches.len(), searches, "the rebuilt collection was not asked"); + assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); + + // Lag it lets through: catching up is the worker's, before the search is taken. + let mut lagging = projected(&embedder, &index, &entries[..3]); + lagging.want(entries[3..].iter().cloned()); + assert_eq!(lagging.progress_status(), IndexStatus::Behind { pending: 1 }); + assert!(lagging.search("anything", 3, &[], SEARCH_DEADLINE).is_ok()); +} + +/// A search that fails for its own reasons (the vector store errs, the +/// embedder is down, an identity read times out) fails that search alone: the +/// reconciliation stands, the status stays `Current`, the collection is not +/// listed again, and the next search is asked and answered. +#[test] +fn a_failed_search_fails_that_search_alone() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + type Fault = Box; + let faults: [(&str, Fault); 3] = [ + ("the vector store fails the search", Box::new(|_, index, on| index.state.lock().unwrap().fail_search = on)), + ("the embedder is down", Box::new(|embedder, _, on| embedder.state.lock().unwrap().down = on)), + ("the identity read never answers", Box::new(|embedder, _, on| embedder.state.lock().unwrap().hang_identity = on)), + ]; + for (what, fault) in faults { + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + let listed = index.state.lock().unwrap().listed; + fault(&embedder, &index, true); + assert!(projector.search("anything", 3, &[], Duration::from_millis(50)).is_err(), "{what}"); + assert_eq!(projector.progress_status(), IndexStatus::Current, "{what}"); + assert!(!projector.stale(), "{what}"); + fault(&embedder, &index, false); + assert!(projector.search("anything", 3, &[], SEARCH_DEADLINE).is_ok(), "{what}: the next search is answered"); + assert_eq!(index.state.lock().unwrap().listed, listed, "{what}: the collection was not listed again"); + } + + // In the worker the failed search is answered and nothing else moves. + let (embedder, index) = pair(); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let listed = index.state.lock().unwrap().listed; + index.state.lock().unwrap().fail_search = true; + let ticket = ask_sink(&mut sink, "anything", 3).unwrap(); + let failed = answers(&mut sink, 1); + assert_eq!(failed[0].0, ticket); + assert!(failed[0].1.is_err()); + index.state.lock().unwrap().fail_search = false; + let next = ask_sink(&mut sink, "anything", 3).expect("the index is still current, not being rebuilt"); + let answered = answers(&mut sink, 1); + assert_eq!((answered[0].0, answered[0].1.is_ok()), (next, true)); + assert_eq!(index.state.lock().unwrap().listed, listed, "the worker did not list the collection again"); +} + +/// A model the search finds changed is a rebuild, and its refusal names the +/// state and how many documents are pending, as every other refusal does. +#[test] +fn a_model_changed_refusal_names_the_state_and_the_pending_count() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + projector.want(entries[3..].iter().cloned()); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + let refusal = format!("{:#}", projector.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()); + assert!(refusal.contains("model changed") && refusal.contains("being rebuilt") && refusal.contains("pending: 1"), "{refusal}"); +} + +/// A search asks the store what the collection says about itself: a label for +/// another mind, no label, or no collection is refused before the query is +/// embedded, and the reconciliation is to be redone. +#[test] +fn a_search_re_reads_the_collections_label() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let cases: [Edit; 3] = [ + ( + "another mind's label", + Box::new(|stored| { + let Described::Labelled(meta) = &mut stored.label else { panic!() }; + meta.mind = "mind-commit-another".into(); + }), + ), + ("no label", Box::new(|stored| stored.label = Described::Unlabelled)), + ("another text version", Box::new(|stored| { + let Described::Labelled(meta) = &mut stored.label else { panic!() }; + meta.index_text_version += 1; + })), + ]; + for (what, change) in cases { + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + change(index.state.lock().unwrap().collections.get_mut(&collection()).unwrap()); + let embedded = embedder.texts().len(); + let refusal = format!("{:#}", projector.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()); + assert!(refusal.contains("no longer the one this mind's index was built as"), "{what}: {refusal}"); + assert_eq!(embedder.texts().len(), embedded, "{what}: nothing was embedded"); + assert!(index.state.lock().unwrap().searches.is_empty(), "{what}"); + assert!(projector.stale(), "{what}"); + } + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + index.state.lock().unwrap().collections.clear(); + assert!(projector.search("anything", 3, &[], SEARCH_DEADLINE).is_err(), "no collection"); + assert!(projector.stale()); +} + +/// The model is read again after the query is embedded: a vector made while the +/// model changed is never compared with the collection. +#[test] +fn a_model_that_changes_while_the_query_is_embedded_is_refused() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let index = FakeIndex::default(); + gate.open(); + let mut projector = projected(&embedder, &index, &entries); + gate.shut_again(); + let searching = std::thread::spawn(move || { + let found = projector.search("anything", 3, &[], SEARCH_DEADLINE); + (projector, found) + }); + gate.wait_arrived(1); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + gate.open(); + let (projector, found) = searching.join().unwrap(); + let refusal = format!("{:#}", found.unwrap_err()); + assert!(refusal.contains("while the texts were embedded"), "{refusal}"); + assert!(index.state.lock().unwrap().searches.is_empty(), "the vector was not used"); + assert!(projector.stale()); +} + +/// Each search has a ticket of its own, and its answer comes back once, as the +/// worker gave it: nothing is added to it and nothing is left to be +/// collected twice. +#[test] +fn every_search_has_its_own_ticket_and_its_answer_comes_back_exactly_once() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + index.state.lock().unwrap().hits = + vec![Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }]; + let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let expected: Hits = vec![( + PipelineRef { kind: PipelineKind::Question, id: Short("eureka-state:question:Q1".into()) }, + 0.9, + )]; + let mut tickets = Vec::new(); + for text in ["a", "b", "c"] { + let among = vec!["eureka-state:question:Q1".to_owned()]; + let ticket = >::search(&mut sink, text, 3, among).unwrap(); + assert_eq!(answers(&mut sink, 1), vec![(ticket, Ok(expected.clone()))]); + assert!(collected(&mut sink).is_empty(), "an answer is handed over once"); + tickets.push(ticket); + } + assert_eq!(tickets, [SearchTicket(0), SearchTicket(1), SearchTicket(2)]); +} + +/// The queue of searches is bounded and the asker who does not fit is told. +/// One search is inside the embedder and the queue is full; the next is +/// refused by name, and when the worker has answered some it takes more. +#[test] +fn the_search_queue_is_bounded_and_the_asker_is_told_when_it_is_full() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + ask_sink(&mut sink, "in flight", 3).unwrap(); + gate.wait_arrived(1); + for n in 0..SEARCHES_QUEUED_MAX { + ask_sink(&mut sink, &format!("waiting {n}"), 3).unwrap_or_else(|why| panic!("{n} of {SEARCHES_QUEUED_MAX}: {why}")); + } + let refusal = ask_sink(&mut sink, "one too many", 3).unwrap_err(); + assert!(refusal.contains(&format!("{SEARCHES_QUEUED_MAX} searches waiting")), "{refusal}"); + + gate.open(); + let answered = answers(&mut sink, SEARCHES_QUEUED_MAX + 1); + assert_eq!(answered.len(), SEARCHES_QUEUED_MAX + 1); + assert!(ask_sink(&mut sink, "room again", 3).is_ok()); +} + +/// A search the asker has stopped waiting for is taken out of the queue and +/// never embedded; the ones on either side of it are. +#[test] +fn an_abandoned_search_is_never_embedded() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let first = ask_sink(&mut sink, "first", 3).unwrap(); + gate.wait_arrived(1); + let second = ask_sink(&mut sink, "second", 3).unwrap(); + let third = ask_sink(&mut sink, "third", 3).unwrap(); + abandon(&mut sink, second); + gate.open(); + let answered = answers(&mut sink, 2); + let tickets: Vec = answered.iter().map(|(ticket, _)| *ticket).collect(); + assert_eq!(tickets, [first, third]); + let texts = queries(&embedder); + assert_eq!(texts.len(), 2, "{texts:?}"); + assert!(texts[0].ends_with("Query: first") && texts[1].ends_with("Query: third"), "{texts:?}"); +} + +/// The worker serves one queued search and returns to its writes: three +/// waiting searches leave two after one turn. +#[test] +fn one_queued_search_is_served_per_turn() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a"), queued(2, "b"), queued(3, "c")], false); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert_eq!(guard.searches.iter().map(|queued| queued.ticket).collect::>(), [SearchTicket(2), SearchTicket(3)]); + assert_eq!(guard.found.iter().map(|(ticket, _)| *ticket).collect::>(), [SearchTicket(1)]); +} + +/// A search reads its own writes. Asked while a batch is in the embedder and +/// eight more documents wait, it is accepted, waits behind them, and is +/// answered only once every one is in the collection. +#[test] +fn a_search_waits_for_the_writes_ahead_of_it_and_reads_them() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), quick()); + gate.wait_arrived(1); + let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); + gate.open(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, ticket); + assert!(answered[0].1.is_ok(), "{answered:?}"); + assert_eq!(index.state.lock().unwrap().held_when_searched, [BATCH + 8], "the search saw every document admitted before it"); + sink.wait_status(|status| *status == IndexStatus::Current); +} + +/// The worker takes admissions before the search, and leaves the search queued +/// while the writes ahead of it are unfinished: a search asked with a document +/// still in the inbox absorbs it and waits, and once the index has caught up +/// it is answered. An index being rebuilt is not waited on: the search is taken +/// and refused, naming the state and the pending count. +#[test] +fn the_worker_absorbs_admissions_before_a_search_and_waits_for_lag_but_not_for_a_rebuild() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + let shared = shared_with(vec![entries[3].clone()], None, vec![queued(1, "a")], false); + + serve_search(&mut projector, &shared); + { + let guard = lock(&shared); + assert!(guard.inbox.is_empty(), "the admission ahead of the search was absorbed"); + assert_eq!(guard.searches.len(), 1, "the search waits for it to be written"); + assert!(guard.found.is_empty()); + } + assert_eq!(projector.pending(), 1); + + drain(&mut projector); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(guard.searches.is_empty()); + assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); + assert_eq!(index.state.lock().unwrap().held_when_searched, [4]); + drop(guard); + + // A rebuild: the collection was lost and made again, and every entry is owed. + let (rebuild_embedder, rebuild_index) = pair(); + let mut rebuilding = rebuilding(&rebuild_embedder, &rebuild_index, &entries); + let shared = shared_with(Vec::new(), None, vec![queued(2, "b")], false); + serve_search(&mut rebuilding, &shared); + let guard = lock(&shared); + let [(SearchTicket(2), Err(refusal))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); +} + +/// A collection made again with nothing to refill owes nothing, so the rebuild +/// ends inside `reconcile`: the next entry is lag, and a search waits for it +/// rather than being refused as a rebuild. +#[test] +fn a_rebuild_that_owes_nothing_settles_when_it_is_made() { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into())); + drain(&mut projector); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + assert_eq!(projector.recheck().unwrap(), Advance::Progressed, "the collection is made again under the new digest"); + assert_eq!(projector.pending(), 0); + + projector.want(entries_of(1)); + assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 1 }, "nothing was owed, so nothing is being rebuilt"); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert_eq!(guard.searches.len(), 1, "the search waits for the write"); + assert!(guard.found.is_empty(), "and is not refused: {:?}", guard.found); +} + +/// An identified mind with nothing owed has still to reconcile once: a search +/// waits for that, and is not taken to fail against an index not yet built. +#[test] +fn a_search_waits_for_the_first_reconcile_of_an_identified_mind_owing_nothing() { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); + assert_eq!(projector.pending(), 0); + assert!(projector.stale()); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + serve_search(&mut projector, &shared); + { + let guard = lock(&shared); + assert_eq!(guard.searches.len(), 1, "the search waits for the reconcile"); + assert!(guard.found.is_empty(), "{:?}", guard.found); + } + drain(&mut projector); + serve_search(&mut projector, &shared); + assert!(matches!(lock(&shared).found.as_slice(), [(SearchTicket(1), Ok(_))])); +} + +/// A search that was queued when the worker failed is answered with why, by +/// the worker's own failure path. +#[test] +fn a_search_queued_before_the_worker_fails_is_answered_with_why() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries_of(1), quick()); + gate.wait_arrived(1); + lock(&sink.shared).searches.push_back(queued(5, "anything")); + embedder.state.lock().unwrap().down = true; + gate.open(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, SearchTicket(5)); + let Err(why) = &answered[0].1 else { panic!("{answered:?}") }; + assert!(why.contains("the semantic index is failing") && why.contains("pending: 1") && why.contains("the embedder is down"), "{why}"); +} + +/// While a failure is being reported, the report's pending count is the one +/// the worker holds now, published under the lock that took the entries out of +/// the inbox: never the count from before they moved. +#[test] +fn a_failure_report_carries_the_pending_count_it_has_now() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); + projector.want(entries[..2].iter().cloned()); + let shared = shared_with(entries[2..].to_vec(), None, Vec::new(), false); + let error = StepError::Unavailable(anyhow::anyhow!("down")); + assert!(absorb(&mut projector, &shared, Some((&error, 3)))); + { + let guard = lock(&shared); + assert!(guard.inbox.is_empty()); + assert_eq!(guard.status, IndexStatus::Failing { pending: 4, attempts: 3, error: "down".into() }); + } + assert!(absorb(&mut projector, &shared, None)); + assert_eq!(lock(&shared).status, IndexStatus::Behind { pending: 4 }, "a first build is lag"); + lock(&shared).closed = true; + assert!(!absorb(&mut projector, &shared, None), "a closed sink ends the worker"); +} + +/// The worker's recheck is on its own clock: searches arriving faster than the +/// interval do not keep pushing it out. +#[test] +fn steady_searches_do_not_postpone_the_workers_recheck() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let backoff = Backoff { recheck: Duration::from_millis(200), ..quick() }; + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); + sink.wait_status(|status| *status == IndexStatus::Current); + let listed = index.state.lock().unwrap().listed; + let started = Instant::now(); + while index.state.lock().unwrap().listed < listed + 2 { + assert!(started.elapsed() < Duration::from_secs(5), "the worker never rechecked while searches kept arriving"); + let _ = ask_sink(&mut sink, "anything", 3); + collected(&mut sink); + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// `wait_status` gives up at one deadline, not one per wake: a health that +/// keeps changing does not keep it waiting. +#[test] +fn wait_status_gives_up_at_its_overall_deadline_however_often_the_health_changes() { + let (embedder, index) = pair(); + let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let shared = Arc::clone(&sink.shared); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let ticking = Arc::clone(&stop); + let ticker = std::thread::spawn(move || { + while !ticking.load(std::sync::atomic::Ordering::Relaxed) { + shared.1.notify_all(); + std::thread::sleep(Duration::from_millis(5)); + } + }); + let (tell, told) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let gave_up = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + sink.wait_status_for(Duration::from_millis(300), |status| *status == IndexStatus::Reconciling { pending: 99 }) + })) + .is_err(); + let _ = tell.send(gave_up); + }); + let gave_up = told.recv_timeout(Duration::from_secs(10)); + stop.store(true, std::sync::atomic::Ordering::Relaxed); + ticker.join().unwrap(); + assert_eq!(gave_up, Ok(true), "the wait did not end at its deadline"); +} + +/// An idle worker verifies once per interval, not in a loop: the recheck moves +/// its own deadline when it runs. +#[test] +fn an_idle_worker_rechecks_once_per_interval() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let backoff = Backoff { recheck: Duration::from_millis(100), ..quick() }; + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); + sink.wait_status(|status| *status == IndexStatus::Current); + std::thread::sleep(Duration::from_millis(600)); + let listed = index.state.lock().unwrap().listed; + assert!((2..=40).contains(&listed), "the startup reconciliation and about five rechecks listed the collection {listed} times"); +} + +// The search's rulings: what may be waiting ahead of it, when a rebuild +// refuses, what a first build is, how time is spent, and who owns the deadline. + +/// A search is never taken while a document sits in the inbox. An admission +/// that lands after the inbox was absorbed and before the search was taken (the +/// hook puts one there, where a second lock once left the gap) leaves the +/// search queued; the next turn absorbs it, and the search is answered only +/// once it is written. +#[test] +fn a_search_is_never_taken_while_an_admission_sits_in_the_inbox() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + let late = entries[3].clone(); + AFTER_ABSORB.with(|hook| *hook.borrow_mut() = Some(Box::new(move |guard| guard.inbox.push(late)))); + + serve_search(&mut projector, &shared); + { + let guard = lock(&shared); + assert_eq!(guard.searches.len(), 1, "the search was not taken past the admission"); + assert!(guard.found.is_empty()); + assert_eq!(guard.inbox.len(), 1); + } + serve_search(&mut projector, &shared); + assert_eq!(lock(&shared).searches.len(), 1, "absorbed, but not yet written"); + drain(&mut projector); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); + assert_eq!(index.state.lock().unwrap().held_when_searched, [4], "the search saw the admission"); +} + +/// Through the sink, an admission acknowledged before the search was asked is +/// in the inbox while the worker is inside a batch; the search reads it. +#[test] +fn a_search_reads_an_admission_acknowledged_before_it_was_asked() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(2), quick()); + gate.wait_arrived(1); + let admitted = entries_of(5).split_off(2); + lock(&sink.shared).inbox.extend(admitted); + let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); + gate.open(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, ticket); + assert!(answered[0].1.is_ok(), "{answered:?}"); + assert_eq!(index.state.lock().unwrap().held_when_searched, [5]); +} + +/// A collection made again is a rebuild until every document it owes is +/// written again: searches are refused as rebuilding, however the collection +/// came to be made again, and however many steps or failures the refill takes. +#[test] +fn a_collection_made_again_refuses_searches_until_it_is_refilled() { + let refused = |projector: &mut Projector, pending: u32| { + let why = format!("{:#}", projector.search("anything", 3, &[], SEARCH_DEADLINE).unwrap_err()); + assert!(why.contains("being rebuilt") && why.contains(&format!("pending: {pending}")), "{why}"); + }; + let entries = entries_of(BATCH + 8); + + // Lost while the worker ran. + let (embedder, index) = pair(); + let mut projector = rebuilding(&embedder, &index, &entries); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); + refused(&mut projector, 40); + index.state.lock().unwrap().down = true; + projector.advance().expect_err("the store is down mid-refill"); + index.state.lock().unwrap().down = false; + assert_eq!(projector.advance().unwrap(), Advance::Progressed, "reconciled again, under a label that already fits"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }, "a failure does not end the rebuild"); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }, "half refilled is still a rebuild"); + refused(&mut projector, 8); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert!(projector.search("anything", 3, &[], SEARCH_DEADLINE).is_ok(), "answered once refilled"); + assert_eq!(index.state.lock().unwrap().held_when_searched, [40]); + projector.want(entries_of(1).into_iter().map(|mut entry| { + entry.id.id.0.push_str("-later"); + entry + })); + assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 1 }, "a later admission is lag, not a rebuild"); + + // Made again because the model changed. + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + assert_eq!(projector.recheck().unwrap(), Advance::Progressed, "the change is seen and the collection made again"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); + projector.advance().unwrap(); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }); + refused(&mut projector, 8); + drain(&mut projector); + assert!(projector.search("anything", 3, &[], SEARCH_DEADLINE).is_ok()); +} + +/// Through the worker: a collection lost while the worker runs is made again, +/// and a search asked during the refill is refused as rebuilding, naming how +/// many documents are pending; one asked after it is answered. +#[test] +fn a_search_asked_during_a_refill_is_refused_and_one_after_it_is_answered() { + let gate = Gate::shut(); + gate.open(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let backoff = Backoff { recheck: Duration::from_millis(50), ..quick() }; + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), backoff); + sink.wait_status(|status| *status == IndexStatus::Current); + gate.shut_again(); + index.state.lock().unwrap().collections.remove(&collection()); + gate.wait_arrived(1); + let refusal = ask_sink(&mut sink, "anything", 3).unwrap_err(); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 40"), "{refusal}"); + gate.open(); + sink.wait_status(|status| *status == IndexStatus::Current); + let ticket = ask_sink(&mut sink, "anything", 3).expect("refilled"); + let answered = answers(&mut sink, 1); + assert_eq!((answered[0].0, answered[0].1.is_ok()), (ticket, true)); + assert_eq!(index.state.lock().unwrap().held_when_searched.last(), Some(&40)); +} + +/// A mind whose index has never been built has nothing stale to protect: its +/// first build is lag, whether the search was asked before the worker had +/// looked at the first admission or after. Only a rebuild of an index that +/// exists refuses. +#[test] +fn a_minds_first_build_is_lag_whoever_looks_first() { + for asked_before_the_worker_looked in [true, false] { + let what = if asked_before_the_worker_looked { "asked first" } else { "asked after the worker looked" }; + let (embedder, index) = pair(); + let entries = entries_of(3); + let (mut projector, shared) = if asked_before_the_worker_looked { + let projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + (projector, shared_with(entries, Some(MIND.into()), Vec::new(), false)) + } else { + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries); + let shared = shared_with(Vec::new(), None, Vec::new(), false); + lock(&shared).status = projector.progress_status(); + (projector, shared) + }; + let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline: SEARCH_DEADLINE }; + let ticket = ask_sink(&mut sink, "anything", 3).unwrap_or_else(|why| panic!("{what}: {why}")); + + serve_search(&mut projector, &shared); + assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 3 }, "{what}: not yet reconciled, and lag"); + assert_eq!(lock(&shared).searches.len(), 1, "{what}: the search waits for the first reconciliation"); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + serve_search(&mut projector, &shared); + assert_eq!(lock(&shared).searches.len(), 1, "{what}: and for the first writes"); + drain(&mut projector); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(matches!(guard.found.as_slice(), [(found, Ok(_))] if *found == ticket), "{what}: {:?}", guard.found); + assert_eq!(index.state.lock().unwrap().held_when_searched, [3], "{what}"); + } + + let entries = entries_of(4); + let (embedder, index) = pair(); + projected(&embedder, &index, &entries[..3]); + let mut partial = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + partial.want(entries.iter().cloned()); + partial.advance().unwrap(); + assert_eq!(partial.progress_status(), IndexStatus::Behind { pending: 1 }, "an index found whole at start-up only lags"); + drain(&mut partial); + index.state.lock().unwrap().collections.remove(&collection()); + assert_eq!(partial.recheck().unwrap(), Advance::Progressed); + assert_eq!(partial.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index found at start-up was built: losing it is a rebuild"); + + embedder.state.lock().unwrap().identity.digest = "d2".into(); + let mut replaced = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); + replaced.want(entries); + replaced.advance().unwrap(); + assert_eq!(replaced.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index that exists and is made again is a rebuild"); +} + +/// A mind with no identity and documents owed has nothing to wait for: the +/// search is taken and refused, not left queued for a worker that would spin +/// on it. +#[test] +fn a_search_is_refused_when_documents_are_owed_to_a_mind_with_no_identity() { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + projector.want(entries_of(2)); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(guard.searches.is_empty()); + let [(SearchTicket(1), Err(why))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; + assert!(why.contains("being rebuilt") && why.contains("pending: 2"), "{why}"); + assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); +} + +/// Every call a search makes is bounded by what its time has left, not by the +/// time it was given: after a slow identity read the embed's bound is smaller +/// by what the read took, and the query's by both reads. +#[test] +fn a_later_calls_bound_shrinks_as_the_search_spends_its_time() { + let entries = entries_of(3); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + let delay = Duration::from_millis(150); + let within = Duration::from_secs(10); + { + let mut state = embedder.state.lock().unwrap(); + state.identity_delay = delay; + state.bounds.clear(); + state.identity_bounds.clear(); + } + index.state.lock().unwrap().bounds.clear(); + projector.search("anything", 3, &[], within).unwrap(); + + let embedder = embedder.state.lock().unwrap(); + let index = index.state.lock().unwrap(); + assert!(embedder.identity_bounds[0] <= within); + assert!(embedder.bounds[0] <= within - delay, "the embed came after one slow read: {:?}", embedder.bounds); + assert!(embedder.identity_bounds[1] <= within - delay, "{:?}", embedder.identity_bounds); + let queried = index.bounds.iter().find(|(call, _)| *call == "search").unwrap().1; + assert!(queried <= within - delay * 2, "the query came after two slow reads: {queried:?}"); +} + +/// The deadline has one owner: the sink. It stamps every search with it, and +/// it is what the serve loop is told (`Daemon::search_deadline`), so the loop's +/// wait and the worker's bound are one value. +#[test] +fn the_sink_owns_the_searchs_deadline_and_tells_the_loop() { + let (_root, mind) = mind_with_documents(); + let deadline = Duration::from_secs(7); + let shared = shared_with(Vec::new(), None, Vec::new(), false); + let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline }; + let asked = Instant::now(); + ask_sink(&mut sink, "anything", 3).unwrap(); + let due = lock(&shared).searches.back().expect("no worker took it").due; + assert!(due >= asked + deadline && due <= Instant::now() + deadline, "stamped with the sink's deadline"); + assert_eq!(>::search_deadline(&sink), deadline); + assert_eq!(Daemon::new(mind, sink).search_deadline(), deadline); +} fn ask_sources(sink: &mut WorkerSink, text: &str) -> Result { let query = SourceQuery { text: huginn_mind::eureka_pipeline::Line(text.into()), texts: vec![], top_k: 5, repo: None, front: false }; >::search_sources(sink, &query) } -fn sources_answers(sink: &mut WorkerSink, count: usize) -> Vec<(SearchTicket, Result)> { +fn sources_answers(sink: &mut WorkerSink, count: usize) -> Vec<(SearchTicket, Result)> { let mut all = Vec::new(); eventually("the source searches being answered", || { - all.extend(>::sources_searched(sink)); + all.extend(>::sources_searched(sink).into_iter().map(|(ticket, found)| { + ( + ticket, + match found { + crate::daemon::Found::Sources(hits) => Ok(hits), + crate::daemon::Found::Failed(why) => Err(why), + other => panic!("a source search was answered with another kind: {}", matches!(other, crate::daemon::Found::Chat(_))), + }, + ) + })); all.len() >= count }); all @@ -2081,6 +2090,60 @@ fn a_worker_stuck_on_the_minds_collection_still_answers_a_source_search() { assert_eq!(answered.indexed_under.digest, "d1"); } +/// A chat search rides the same queue and worker as a source search, answers +/// from the same collection while the mind's own collection is stuck, and comes +/// back as chat hits. +#[test] +fn a_worker_answers_a_chat_search_from_the_shared_collection() { + use crate::sources::{ChatPayload, ChatPoint, DescribedSources, SourcesMeta}; + use huginn_mind::ChatMessage; + let (embedder, index) = pair(); + index.plant( + &collection_name(&slug(INSTANCE)), + Described::Labelled(CollectionMeta { + managed_by: MANAGED_BY.into(), + instance: INSTANCE.into(), + mind: "another-mind".into(), + model: "model".into(), + model_digest: "d1".into(), + dimensions: 4, + index_text_version: INDEX_TEXT_VERSION, + }), + ); + let meta = SourcesMeta { managed_by: MANAGED_BY.into(), corpus: crate::sources::CORPUS.into(), model: "model".into(), model_digest: "d1".into(), dimensions: 4, chunker_version: 1 }; + index.sources.plant(DescribedSources::Labelled { meta, cursors: Default::default() }); + index + .sources + .state + .lock() + .unwrap() + .chat + .insert( + "c1".into(), + ChatPoint { + id: "c1".into(), + vector: vec![1.0; 4], + payload: ChatPayload { + room_id: "!r:x".into(), + room_alias: None, + messages: vec![ChatMessage { event_id: "$e".into(), sender: "@a:x".into(), ts_ms: 5, body: "hi".into() }], + text: "t".into(), + }, + }, + ); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(1), quick()); + sink.wait_status(|status| matches!(status, IndexStatus::Refused { .. })); + let query = huginn_mind::ChatQuery { texts: vec![], rooms: vec![], after_ms: None, before_ms: None, top_k: 5 }; + let ticket = >::search_chat(&mut sink, &query).unwrap(); + let mut found = Vec::new(); + eventually("the chat search being answered", || { + found.extend(>::sources_searched(&mut sink)); + !found.is_empty() + }); + let (answered, Found::Chat(hits)) = found.remove(0) else { panic!("a chat ask is answered with chat hits") }; + assert_eq!((answered, hits.hits.len(), hits.hits[0].first_event_id.as_str(), hits.truncated), (ticket, 1, "$e", false)); +} + /// Source searches take their own tickets, share the queue's bound with the /// mind's searches, and an abandon removes the one ticket named from either. #[test] diff --git a/crates/huginn-daemon/src/sources/mod.rs b/crates/huginn-daemon/src/sources/mod.rs index 24a16c2..0189aba 100644 --- a/crates/huginn-daemon/src/sources/mod.rs +++ b/crates/huginn-daemon/src/sources/mod.rs @@ -1,5 +1,7 @@ -//! The source index: the public GameCult forge repos' code and docs as chunks -//! in one Qdrant collection, `huginn_sources`, for semantic search. +//! The source index: the public GameCult forge repos' code and docs, and the +//! Minds' chat rooms, as chunks in one Qdrant collection, `huginn_sources`, +//! for semantic search. Each point's `corpus` payload (`forge` or `chat`) keeps +//! the two apart; a forge search filters `forge` and a chat search `chat`. //! //! Two truths, one writer. The forge is the truth of which repos exist and what //! they hold; the collection is derived from it and can be deleted whenever. @@ -30,7 +32,7 @@ use std::path::Path; use std::time::{Duration, Instant}; use anyhow::{Context, Result, bail, ensure}; -use huginn_mind::{SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery}; +use huginn_mind::{ChatHit, ChatHits, ChatMessage, ChatQuery, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery}; use crate::index::{BATCH, BULK_EMBED_TIMEOUT, Budget, Embedder, INDEX_CALL_TIMEOUT, MANAGED_BY, ModelIdentity, point_id}; use chunk::{CHUNKER_VERSION, embedded_text}; @@ -43,8 +45,19 @@ pub const COLLECTION: &str = "huginn_sources"; /// agent reads first to learn what the repo is. The only owner of the list. pub const FRONT_PAGE_FILES: [&str; 6] = ["README.md", "README", "readme.md", "README.markdown", "CLAUDE.md", "AGENTS.md"]; -/// What the collection says its corpus is. -pub const CORPUS: &str = "forge:GameCult"; +/// What the collection says its corpus is: the set of corpora it holds, each +/// point's `corpus` payload naming which (`forge` or `chat`). A collection +/// labelled with any other set is rebuilt. +pub const CORPUS: &str = "forge:GameCult+chat:rooms"; + +/// The `corpus` payload of a forge point, and of a chat point. +pub const CORPUS_FORGE: &str = "forge"; +pub const CORPUS_CHAT: &str = "chat"; + +/// The instruction Qwen3 embedding models take in front of a question about +/// past conversation. +pub const CHAT_QUERY_INSTRUCTION: &str = + "Given a question about past conversation in chat rooms, retrieve the chat passages that answer it"; /// The instruction Qwen3 embedding models take in front of a query about code. pub const SOURCE_QUERY_INSTRUCTION: &str = @@ -111,6 +124,58 @@ pub struct SourcePoint { pub payload: SourcePayload, } +/// What travels beside a chat chunk's vector: consecutive messages of one +/// room, never spanning rooms. The single owner of the chat point's shape; the +/// ingest job is its only writer (through `upsert_chat`) and nothing else of the +/// payload is read. `messages` are the display truth and hold, in order, at +/// least one message. The chunk's span, its first and last event ids and its +/// `event_ids` are derived from them by the codec, so they cannot disagree. +/// `text` is what was embedded: the room's name, then `time sender: body` per +/// message. +#[derive(Clone, Debug, PartialEq)] +pub struct ChatPayload { + pub room_id: String, + /// Absent when the room has no canonical alias. + pub room_alias: Option, + pub messages: Vec, + pub text: String, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ChatPoint { + pub id: String, + pub vector: Vec, + pub payload: ChatPayload, +} + +/// One chat chunk a vector search returned and how near it scored. +#[derive(Clone, Debug, PartialEq)] +pub struct ChatRow { + pub score: f32, + pub payload: ChatPayload, +} + +/// Which chat chunks a read keeps; the selectors compose as AND. `rooms` are +/// any-of room ids (`!`) or aliases (`#`); the span keeps chunks whose own +/// span overlaps [`after_ms`, `before_ms`]. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ChatFilter { + pub rooms: Vec, + pub after_ms: Option, + pub before_ms: Option, +} + +impl ChatFilter { + /// Whether a chunk of `room_id`/`room_alias` spanning [`start`, `end`] + /// passes. The fake index and the tests read the filter through this; the + /// Qdrant index states the same thing as conditions. + pub fn keeps(&self, room_id: &str, room_alias: Option<&str>, start: u64, end: u64) -> bool { + (self.rooms.is_empty() || self.rooms.iter().any(|room| room == room_id || Some(room.as_str()) == room_alias)) + && self.after_ms.is_none_or(|after| end >= after) + && self.before_ms.is_none_or(|before| start <= before) + } +} + /// One chunk a vector search returned and how near it scored. #[derive(Clone, Debug, PartialEq)] pub struct SourceRow { @@ -124,7 +189,9 @@ pub struct SourceRow { pub trait SourceIndex { fn describe_sources(&mut self, within: Duration) -> Result; /// Drops the collection if it exists and creates it empty with this - /// metadata, no cursors, and keyword indexes on `repo` and `path`. + /// metadata, no cursors, keyword indexes on `repo`, `path`, `corpus`, + /// `room_id`, `room_alias` and `event_ids`, and integer indexes on + /// `ts_start_ms` and `ts_end_ms`. fn recreate_sources(&mut self, meta: &SourcesMeta) -> Result<()>; /// Replaces the cursors object whole. fn set_cursors(&mut self, cursors: &BTreeMap) -> Result<()>; @@ -137,6 +204,15 @@ pub trait SourceIndex { /// The `limit` nearest chunks, best first, in `repo` alone when given, and /// among the `FRONT_PAGE_FILES` alone when `front`. fn search_sources(&mut self, vector: &[f32], repo: Option<&str>, front: bool, limit: u32, within: Duration) -> Result>; + /// Writes chat chunks, acknowledged before it returns. The ingest job's + /// alone: the daemon never calls it. A chunk with no messages is refused. + fn upsert_chat(&mut self, points: &[ChatPoint]) -> Result<()>; + /// The `limit` chat chunks nearest `vector`, best first, among those + /// `filter` keeps. Reads only `corpus=chat` points; the daemon's. + fn chat_nearest(&mut self, vector: &[f32], filter: &ChatFilter, limit: u32, within: Duration) -> Result>; + /// Up to `limit` chat chunks `filter` keeps, oldest first by span start. + /// Reads only `corpus=chat` points; the daemon's. + fn chat_scroll(&mut self, filter: &ChatFilter, limit: u32, within: Duration) -> Result>; } /// A repo the forge lists as indexable. @@ -476,6 +552,10 @@ pub fn fuse(lists: Vec>, key: impl Fn(&T) -> String, k: u32, top_k: us /// of one, trimmed, and each distinct text once, in the order asked. fn query_texts(query: &SourceQuery) -> Vec<&str> { let asked: Vec<&str> = if query.texts.is_empty() { vec![query.text.0.as_str()] } else { query.texts.iter().map(|text| text.0.as_str()).collect() }; + distinct_texts(asked) +} + +fn distinct_texts(asked: Vec<&str>) -> Vec<&str> { let mut distinct: Vec<&str> = Vec::with_capacity(asked.len()); for text in asked { if !distinct.iter().any(|seen| seen.trim() == text.trim()) { @@ -485,16 +565,12 @@ fn query_texts(query: &SourceQuery) -> Vec<&str> { distinct } -/// A source search, which a daemon answers from the collection as it finds it: -/// it embeds the query with its own embedder, and refuses if the collection -/// was built under another model, because vectors of two models do not compare. -pub fn search(embedder: &mut E, index: &mut S, query: &SourceQuery, within: Duration) -> Result { - let budget = Budget::new(within); - let built = match index.describe_sources(budget.left()?).context("describing the source collection")? { - DescribedSources::Absent => bail!("source index not built"), - DescribedSources::Unlabelled => bail!("source index not built: {COLLECTION} carries no Huginn metadata"), - DescribedSources::Labelled { meta, .. } => meta, - }; +/// The label of the collection a daemon is about to read, if it was built +/// under the model this daemon embeds with. Absent, unlabelled and other-model +/// collections are loud refusals: vectors of two models do not compare, and an +/// empty page is not an answer. +fn readable_under_this_model(embedder: &mut E, index: &mut S, budget: &Budget) -> Result<(SourcesMeta, ModelIdentity)> { + let built = built_label(index, budget)?; let identity = embedder.model_identity(budget.left()?).context("reading the embedding model's identity")?; ensure!( built.model == identity.name && built.model_digest == identity.digest && built.dimensions == identity.dimensions, @@ -506,6 +582,23 @@ pub fn search(embedder: &mut E, index: &mut S, quer identity.digest, identity.dimensions ); + Ok((built, identity)) +} + +fn built_label(index: &mut S, budget: &Budget) -> Result { + match index.describe_sources(budget.left()?).context("describing the source collection")? { + DescribedSources::Absent => bail!("source index not built"), + DescribedSources::Unlabelled => bail!("source index not built: {COLLECTION} carries no Huginn metadata"), + DescribedSources::Labelled { meta, .. } => Ok(meta), + } +} + +/// A source search, which a daemon answers from the collection as it finds it: +/// it embeds the query with its own embedder, and refuses if the collection +/// was built under another model, because vectors of two models do not compare. +pub fn search(embedder: &mut E, index: &mut S, query: &SourceQuery, within: Duration) -> Result { + let budget = Budget::new(within); + let (built, identity) = readable_under_this_model(embedder, index, &budget)?; let topics = query_texts(query); let prompts: Vec = topics.iter().map(|topic| format!("Instruct: {SOURCE_QUERY_INSTRUCTION}\nQuery: {topic}")).collect(); let vectors = embedder.embed(&prompts, budget.left()?).context("embedding the query")?; @@ -539,3 +632,68 @@ pub fn search(embedder: &mut E, index: &mut S, quer .collect(), }) } + +/// A chat search. With texts, the chunks nearest them among those the room and +/// span selectors keep, several texts fused by `fuse`; with none, a listing of +/// what the selectors keep, oldest first, `top_k` of it and `truncated` saying +/// more exist (no embedding, so the model's identity is not consulted, though +/// an absent collection is still refused). A hit outside the span is not +/// returned and a boundary hit loses its messages outside [after, before], its +/// span and first and last ids recomputed from what is left. Never writes. +pub fn search_chat(embedder: &mut E, index: &mut S, query: &ChatQuery, within: Duration) -> Result { + let budget = Budget::new(within); + let filter = ChatFilter { rooms: query.rooms.iter().map(|room| room.0.clone()).collect(), after_ms: query.after_ms, before_ms: query.before_ms }; + let top_k = query.top_k as usize; + let topics = distinct_texts(query.texts.iter().map(|text| text.0.as_str()).collect()); + let (built, rows, truncated) = if topics.is_empty() { + let built = built_label(index, &budget)?; + let mut held = index.chat_scroll(&filter, query.top_k + 1, budget.left()?)?; + let truncated = held.len() > top_k; + held.truncate(top_k); + (built, held.into_iter().map(|payload| (None, payload)).collect::>(), truncated) + } else { + let (built, identity) = readable_under_this_model(embedder, index, &budget)?; + let prompts: Vec = topics.iter().map(|topic| format!("Instruct: {CHAT_QUERY_INSTRUCTION}\nQuery: {topic}")).collect(); + let vectors = embedder.embed(&prompts, budget.left()?).context("embedding the query")?; + ensure!(vectors.len() == prompts.len(), "the embedder returned {} vectors for {} query texts", vectors.len(), prompts.len()); + let depth = if vectors.len() == 1 { query.top_k } else { FUSE_DEPTH }; + let mut lists = Vec::with_capacity(vectors.len()); + for vector in &vectors { + ensure!(vector.len() == identity.dimensions as usize, "the embedder returned a query vector of {} where the collection holds {}", vector.len(), identity.dimensions); + lists.push(index.chat_nearest(vector, &filter, depth, budget.left()?)?); + } + let rows: Vec = if lists.len() == 1 { + lists.remove(0) + } else { + let key = |row: &ChatRow| format!("{}\0{}", row.payload.room_id, row.payload.messages.first().map_or("", |message| message.event_id.as_str())); + fuse(lists, key, FUSE_K, top_k).into_iter().map(|(score, row)| ChatRow { score, ..row }).collect() + }; + (built, rows.into_iter().map(|row| (Some(row.score), row.payload)).collect(), false) + }; + Ok(ChatHits { + indexed_under: SourceModel { name: built.model, digest: built.model_digest }, + truncated, + hits: rows.into_iter().filter_map(|(score, payload)| chat_hit(score, payload, &filter)).collect(), + }) +} + +/// A held chunk as a hit: only the messages inside the asked span, the span and +/// ids those messages make, and nothing at all when none is inside. +fn chat_hit(score: Option, payload: ChatPayload, filter: &ChatFilter) -> Option { + let messages: Vec = payload + .messages + .into_iter() + .filter(|message| filter.after_ms.is_none_or(|after| message.ts_ms >= after) && filter.before_ms.is_none_or(|before| message.ts_ms <= before)) + .collect(); + let (first, last) = (messages.first()?, messages.last()?); + Some(ChatHit { + room_id: payload.room_id, + room_alias: payload.room_alias, + first_event_id: first.event_id.clone(), + last_event_id: last.event_id.clone(), + ts_start_ms: messages.iter().map(|message| message.ts_ms).min()?, + ts_end_ms: messages.iter().map(|message| message.ts_ms).max()?, + score, + messages, + }) +} diff --git a/crates/huginn-daemon/src/sources/tests.rs b/crates/huginn-daemon/src/sources/tests.rs index bda5ce9..e46554e 100644 --- a/crates/huginn-daemon/src/sources/tests.rs +++ b/crates/huginn-daemon/src/sources/tests.rs @@ -7,7 +7,7 @@ use std::process::Command; use anyhow::Result; use huginn_mind::eureka_pipeline::{Line, Short}; -use huginn_mind::SourceQuery; +use huginn_mind::{ChatHits, ChatQuery, SourceQuery}; use super::*; use crate::index::fakes::{FakeEmbedder, FakeSources}; @@ -755,3 +755,184 @@ fn a_long_brief_of_eight_topics_reaches_the_chunk_only_the_last_is_near() { assert!(paths(&found).contains(&"chunk-7.rs"), "{:?}", paths(&found)); assert_eq!(paths(&found).len(), 8); } + +// ---- chat search ---- + +fn msg(event_id: &str, ts_ms: u64) -> ChatMessage { + ChatMessage { event_id: event_id.into(), sender: "@a:gamecult.org".into(), ts_ms, body: format!("body of {event_id}") } +} + +/// A chunk in `room` (with `alias`) of messages `(id, ts)`, near `axis`. +fn chat_point(room: &str, alias: Option<&str>, axis: usize, dimensions: u32, messages: &[(&str, u64)]) -> ChatPoint { + let messages: Vec = messages.iter().map(|(id, ts)| msg(id, *ts)).collect(); + ChatPoint { + id: point_id(&format!("{room}{}", messages[0].event_id)), + vector: (0..dimensions as usize).map(|at| if at == axis { 1.0 } else { 0.0 }).collect(), + payload: ChatPayload { room_id: room.into(), room_alias: alias.map(str::to_owned), messages, text: format!("chunk near topic-{axis}") }, + } +} + +fn chat_query(texts: &[&str], rooms: &[&str], after: Option, before: Option, top_k: u32) -> ChatQuery { + ChatQuery { + texts: texts.iter().map(|text| Line((*text).into())).collect(), + rooms: rooms.iter().map(|room| Short((*room).into())).collect(), + after_ms: after, + before_ms: before, + top_k, + } +} + +/// A labelled collection that also holds the chat chunks given. Two rooms: +/// `!a` (alias `#swarm`), `!b` (no alias). +fn chat_rig(points: Vec) -> (Rig, TopicEmbedder) { + let (mut rig, embedder) = topic_rig(4); + rig.index.state.lock().unwrap().points.clear(); + rig.index.upsert_chat(&points).unwrap(); + (rig, embedder) +} + +fn ids(hits: &ChatHits) -> Vec<&str> { + hits.hits.iter().map(|hit| hit.first_event_id.as_str()).collect() +} + +const WITHIN: Duration = Duration::from_secs(5); + +fn three_rooms() -> Vec { + vec![ + chat_point("!a:x", Some("#swarm:x"), 0, 4, &[("$a1", 100), ("$a2", 200)]), + chat_point("!a:x", Some("#swarm:x"), 1, 4, &[("$a3", 300), ("$a4", 400)]), + chat_point("!b:x", None, 0, 4, &[("$b1", 150)]), + ] +} + +#[test] +fn a_room_filter_by_id_and_by_alias_returns_only_that_room() { + let (mut rig, mut embedder) = chat_rig(three_rooms()); + let by_alias = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], None, None, 10), WITHIN).unwrap(); + assert_eq!(ids(&by_alias), ["$a1", "$a3"]); + let by_id = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["!b:x"], None, None, 10), WITHIN).unwrap(); + assert_eq!(ids(&by_id), ["$b1"], "a room with no alias is reachable by id"); + assert_eq!(by_id.hits[0].room_alias, None); + let both = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x", "!b:x"], None, None, 10), WITHIN).unwrap(); + assert_eq!(ids(&both), ["$a1", "$b1", "$a3"], "any-of, oldest first"); + let nobody = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#nobody:x"], None, None, 10), WITHIN).unwrap(); + assert!(nobody.hits.is_empty() && !nobody.truncated, "an unknown alias is zero hits, not an error"); +} + +#[test] +fn a_time_span_returns_the_overlapping_chunks_and_trims_boundary_messages() { + let (mut rig, mut embedder) = chat_rig(three_rooms()); + let hits = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], Some(150), Some(350), 10), WITHIN).unwrap(); + assert_eq!(ids(&hits), ["$a2", "$a3"], "the boundary chunks keep only in-span messages and first ids are recomputed"); + let (first, second) = (&hits.hits[0], &hits.hits[1]); + assert_eq!((first.ts_start_ms, first.ts_end_ms, first.last_event_id.as_str(), first.messages.len()), (200, 200, "$a2", 1)); + assert_eq!((second.ts_start_ms, second.ts_end_ms, second.last_event_id.as_str(), second.messages.len()), (300, 300, "$a3", 1)); + let none = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["#swarm:x"], Some(1000), None, 10), WITHIN).unwrap(); + assert!(none.hits.is_empty(), "a chunk wholly outside is absent"); +} + +#[test] +fn list_mode_is_oldest_first_truncated_when_more_exist_and_needs_no_embed() { + let (mut rig, mut embedder) = chat_rig(three_rooms()); + let page = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &[], None, None, 2), WITHIN).unwrap(); + assert_eq!(ids(&page), ["$a1", "$b1"]); + assert!(page.truncated); + assert!(embedder.0.state.lock().unwrap().embedded.is_empty(), "no texts, no embed call"); + // Within one room chunks never overlap, so the last hit's end + 1 continues without a repeat. + let first = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["!a:x"], None, None, 1), WITHIN).unwrap(); + assert_eq!((ids(&first), first.truncated), (vec!["$a1"], true)); + let rest = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &["!a:x"], Some(first.hits[0].ts_end_ms + 1), None, 1), WITHIN).unwrap(); + assert_eq!((ids(&rest), rest.truncated), (vec!["$a3"], false)); +} + +#[test] +fn texts_rank_by_meaning_and_compose_with_room_and_span() { + let (mut rig, mut embedder) = chat_rig(three_rooms()); + let near = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0"], &[], None, None, 5), WITHIN).unwrap(); + assert_eq!(near.hits.len(), 3); + assert!(near.hits[0].score.unwrap() > near.hits[2].score.unwrap()); + assert_eq!(near.hits[2].first_event_id, "$a3", "the far chunk is last"); + let narrowed = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0"], &["!a:x"], Some(250), None, 5), WITHIN).unwrap(); + assert_eq!(ids(&narrowed), ["$a3"], "a near chunk in another room or outside the span never appears"); + let fused = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0", "topic-1"], &["!a:x"], None, None, 5), WITHIN).unwrap(); + assert_eq!(fused.hits.len(), 2); + assert!(fused.hits[0].score.unwrap() > 0.0 && fused.hits[0].score != Some(1.0), "two texts carry the fused reciprocal-rank score"); + assert_eq!(embedder.0.state.lock().unwrap().embedded.last().unwrap().len(), 2, "one embed call for two texts"); +} + +#[test] +fn every_hit_cites_room_event_ids_and_timestamps() { + let (mut rig, mut embedder) = chat_rig(three_rooms()); + for texts in [&[][..], &["topic-0"][..]] { + let hits = search_chat(&mut embedder, &mut rig.index, &chat_query(texts, &[], None, None, 10), WITHIN).unwrap(); + assert!(!hits.hits.is_empty()); + for hit in &hits.hits { + assert!(hit.room_id.starts_with('!') && !hit.first_event_id.is_empty() && !hit.last_event_id.is_empty()); + assert!(hit.ts_start_ms <= hit.ts_end_ms && !hit.messages.is_empty()); + assert_eq!((hit.messages[0].event_id.as_str(), hit.messages.last().unwrap().event_id.as_str()), (hit.first_event_id.as_str(), hit.last_event_id.as_str())); + for message in &hit.messages { + assert!(message.event_id.starts_with('$') && message.sender.starts_with('@') && message.ts_ms > 0); + } + } + } +} + +#[test] +fn a_forge_search_never_returns_a_chat_point_and_a_chat_search_never_a_forge_point() { + let (mut rig, mut embedder) = topic_rig(4); + rig.index.upsert_chat(&three_rooms()).unwrap(); + let forge = search(&mut embedder, &mut rig.index, &queries(&["topic-0"], 10), WITHIN).unwrap(); + assert_eq!(forge.hits.len(), 4, "only the four forge chunks"); + let chat = search_chat(&mut embedder, &mut rig.index, &chat_query(&["topic-0"], &[], None, None, 10), WITHIN).unwrap(); + assert_eq!(chat.hits.len(), 3, "only the three chat chunks"); + let listed = search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &[], None, None, 10), WITHIN).unwrap(); + assert_eq!(listed.hits.len(), 3); +} + +#[test] +fn an_absent_unlabelled_or_other_model_collection_is_a_loud_refusal_for_chat() { + let mut rig = Rig::new(); + let mut embedder = TopicEmbedder(FakeEmbedder::new("d1")); + for texts in [&[][..], &["topic-0"][..]] { + let error = search_chat(&mut embedder, &mut rig.index, &chat_query(texts, &[], None, None, 3), WITHIN).unwrap_err(); + assert_eq!(format!("{error:#}"), "source index not built"); + } + rig.index.plant(DescribedSources::Unlabelled); + assert!(search_chat(&mut embedder, &mut rig.index, &chat_query(&[], &[], None, None, 3), WITHIN).is_err()); + let (mut rig, _) = chat_rig(three_rooms()); + let mut foreign = TopicEmbedder(FakeEmbedder::new("d2")); + let error = format!("{:#}", search_chat(&mut foreign, &mut rig.index, &chat_query(&["topic-0"], &[], None, None, 3), WITHIN).unwrap_err()); + assert!(error.contains("built under model d1") && error.contains("embeds under model d2"), "{error}"); +} + +/// An old forge-only collection (labelled with the old corpus) is rebuilt once +/// under the new corpus, and a second refresh leaves it. +#[test] +fn an_old_forge_only_collection_rebuilds_once() { + let remote = Remote::new(); + remote.commit(&[("a.rs", b"one\n")], &[]); + let mut rig = Rig::new(); + let mut forge = FakeForge::of(&[("Demo", &remote)]); + rig.converged(&mut forge); + let mut old = match rig.index.state.lock().unwrap().described.clone() { + Some(DescribedSources::Labelled { meta, cursors }) => (meta, cursors), + other => panic!("{other:?}"), + }; + assert_eq!(old.0.corpus, "forge:GameCult+chat:rooms"); + old.0.corpus = "forge:GameCult".into(); + rig.index.plant(DescribedSources::Labelled { meta: old.0, cursors: old.1 }); + let before = rig.index.state.lock().unwrap().recreated; + rig.converged(&mut forge); + rig.converged(&mut forge); + assert_eq!(rig.index.state.lock().unwrap().recreated, before + 1); +} + +#[test] +fn the_chat_read_path_holds_no_discord_matrix_or_second_fuse() { + let source = include_str!("mod.rs"); + let read = &source[source.find("pub fn search_chat").unwrap()..]; + for banned in ["discord", "puppet", "bridge", "matrix", "synapse", "access_token"] { + assert!(!read.to_ascii_lowercase().contains(banned), "{banned} in the chat read path"); + } + assert_eq!(source.matches("pub fn fuse").count(), 1); +} diff --git a/crates/huginn-mind/src/lib.rs b/crates/huginn-mind/src/lib.rs index 3ae57fc..abcd46e 100644 --- a/crates/huginn-mind/src/lib.rs +++ b/crates/huginn-mind/src/lib.rs @@ -72,7 +72,7 @@ pub use store::{MindStore, OwnedRedbMessagePackBackingStore}; pub use wire::{ DeferredAnswer, HuginnMindRequest, HuginnMindResponse, IndexStatus, MAX_DEFERRED_BODY_BYTES, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON, MIND_RESPONSE_SCHEMA, MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, MindStatus, PersonaStatus, - ACTIVITY_REPOS_MAX, OpenPr, PrAction, RepoActivity, RepoChange, RepoEvent, RepoMoved, SOURCE_TEXTS_MAX, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery, + ACTIVITY_REPOS_MAX, CHAT_ROOMS_MAX, ChatHit, ChatHits, ChatMessage, ChatQuery, OpenPr, PrAction, RepoActivity, RepoChange, RepoEvent, RepoMoved, SOURCE_TEXTS_MAX, SOURCE_TOP_K_MAX, SourceHit, SourceHits, SourceModel, SourceQuery, }; /// The leaf, whole, through the one crate that pins its rev: the daemon and diff --git a/crates/huginn-mind/src/wire.rs b/crates/huginn-mind/src/wire.rs index 3f64def..7952f07 100644 --- a/crates/huginn-mind/src/wire.rs +++ b/crates/huginn-mind/src/wire.rs @@ -92,6 +92,10 @@ pub enum HuginnMindRequest { /// docs, which no mind holds. The daemon embeds the text and asks the /// index it does not write. SearchSources { instance: Slug, query: SourceQuery }, + /// Search of the chat index: chunks of the rooms the Minds are meant to + /// have, by room, by time span and by meaning. Read-only, like + /// `SearchSources`, and queued with it. + SearchChat { instance: Slug, query: ChatQuery }, /// What others did in `repos` on the forge since the `after` cursor: the /// daemon reads the forge and no document of the mind, and answers /// `Refused(Unavailable)` when it has no forge or the forge does not answer @@ -270,6 +274,96 @@ pub struct SourceHits { pub hits: Vec, } +/// The most rooms one chat search names. +pub const CHAT_ROOMS_MAX: usize = 16; + +/// A chat search. The selectors compose as AND. `rooms` are any-of: a value +/// beginning `!` is a room id, `#` a canonical alias. `after_ms` and +/// `before_ms` (Unix milliseconds) keep the chunks whose span overlaps +/// [after, before]. `texts` (none to `SOURCE_TEXTS_MAX`) is the meaning match, +/// several fused as in a source search; with none the search is a listing: +/// chunks oldest first, at most `top_k`, `ChatHits.truncated` saying more +/// exist, and the caller narrows with `after_ms` = the last hit's +/// `ts_end_ms + 1`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ChatQuery { + pub texts: Vec, + pub rooms: Vec, + pub after_ms: Option, + pub before_ms: Option, + pub top_k: u32, +} + +impl ChatQuery { + /// The door. A refusal names the field. + pub fn check(&self) -> Result<(), MindRefusal> { + let refused = |field: &str, value: Option, message: &str| MindRefusal::SelectionInvalid { + field: field.into(), + value, + message: message.into(), + }; + if self.texts.len() > SOURCE_TEXTS_MAX { + return Err(refused("texts", Some(self.texts.len().to_string()), "a chat search takes 0 to 8 texts")); + } + if self.texts.iter().any(|text| text.0.trim().is_empty()) { + return Err(refused("texts", None, "every text of a chat search says something")); + } + if self.rooms.len() > CHAT_ROOMS_MAX { + return Err(refused("rooms", Some(self.rooms.len().to_string()), "a chat search names at most 16 rooms")); + } + if let Some(room) = self.rooms.iter().find(|room| !(room.0.starts_with('!') || room.0.starts_with('#')) || room.0.len() < 2) { + return Err(refused("rooms", Some(room.0.clone()), "a room is a room id beginning `!` or a canonical alias beginning `#`")); + } + if let (Some(after), Some(before)) = (self.after_ms, self.before_ms) + && before < after + { + return Err(refused("before_ms", Some(before.to_string()), "before_ms is not earlier than after_ms")); + } + if !(1..=SOURCE_TOP_K_MAX).contains(&self.top_k) { + return Err(refused("top_k", Some(self.top_k.to_string()), "a chat search asks for 1 to 50 hits")); + } + Ok(()) + } +} + +/// One message of a chunk, as the room held it. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct ChatMessage { + pub event_id: String, + /// The sender's Matrix id. + pub sender: String, + pub ts_ms: u64, + pub body: String, +} + +/// One chunk of consecutive messages in one room. Every citation is +/// required: only the room's alias may be absent, because a room may have +/// none. The span and the first and last ids are those of `messages` as +/// returned, after any trimming to the asked span. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ChatHit { + pub room_id: String, + pub room_alias: Option, + pub first_event_id: String, + pub last_event_id: String, + pub ts_start_ms: u64, + pub ts_end_ms: u64, + /// Cosine similarity for one text, the reciprocal-rank sum for several, + /// none in a listing. + pub score: Option, + pub messages: Vec, +} + +/// The answer to a chat search: oldest first in a listing, nearest first with +/// texts. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ChatHits { + pub indexed_under: SourceModel, + /// A listing had more chunks than `top_k`. + pub truncated: bool, + pub hits: Vec, +} + impl HuginnMindRequest { /// The envelope's `operation`. pub fn operation(&self) -> &'static str { @@ -281,6 +375,7 @@ impl HuginnMindRequest { Self::PersonaGet { .. } => "persona_get", Self::PersonaPut { .. } => "persona_put", Self::SearchSources { .. } => "search_sources", + Self::SearchChat { .. } => "search_chat", Self::StewardedActivity { .. } => "stewarded_activity", Self::RootsTouched { .. } => "roots_touched", } @@ -296,6 +391,7 @@ impl HuginnMindRequest { | Self::PersonaGet { instance } | Self::PersonaPut { instance, .. } | Self::SearchSources { instance, .. } + | Self::SearchChat { instance, .. } | Self::StewardedActivity { instance, .. } | Self::RootsTouched { instance } => Some(instance), } @@ -315,6 +411,7 @@ pub enum HuginnMindResponse { Persona(Option), PersonaPut(PersonaPutOutcome), Sources(SourceHits), + Chat(ChatHits), Activity(RepoActivity), RootsTouched(Vec), /// The answer did not fit one send and is being held for the client to @@ -450,6 +547,10 @@ mod tests { instance: slug(INSTANCE), query: SourceQuery { text: Line::default(), texts: vec![Line("reconcile".into()), Line("fuse".into())], top_k: 5, repo: None, front: false }, }, + HuginnMindRequest::SearchChat { + instance: slug(INSTANCE), + query: ChatQuery { texts: vec![Line("release".into())], rooms: vec![Short("#swarm:gamecult.org".into())], after_ms: Some(1), before_ms: None, top_k: 5 }, + }, HuginnMindRequest::RootsTouched { instance: slug(INSTANCE) }, ] } @@ -500,6 +601,20 @@ mod tests { text: "x".into(), }], }), + HuginnMindResponse::Chat(ChatHits { + indexed_under: SourceModel { name: "m".into(), digest: "d".into() }, + truncated: false, + hits: vec![ChatHit { + room_id: "!r:gamecult.org".into(), + room_alias: None, + first_event_id: "$a".into(), + last_event_id: "$a".into(), + ts_start_ms: 1, + ts_end_ms: 1, + score: None, + messages: vec![ChatMessage { event_id: "$a".into(), sender: "@x:gamecult.org".into(), ts_ms: 1, body: "hi".into() }], + }], + }), HuginnMindResponse::RootsTouched(seeded().roots_touched().unwrap()), HuginnMindResponse::Deferred(DeferredAnswer { manifest: cultnet_rs::pack_content("huginn.mind_response", "package", "", "", "", b"body", 4) @@ -515,7 +630,7 @@ mod tests { #[test] fn the_wire_vocabulary_is_the_minds_methods_and_status_is_derived_from_the_response() { let names: Vec<&str> = requests().iter().map(|request| request.operation()).collect(); - assert_eq!(names, ["whoami", "admit", "view", "query", "persona_get", "persona_put", "search_sources", "search_sources", "roots_touched"]); + assert_eq!(names, ["whoami", "admit", "view", "query", "persona_get", "persona_put", "search_sources", "search_sources", "search_chat", "roots_touched"]); for request in requests() { let declared = request.instance().is_none(); assert_eq!(declared, matches!(request, HuginnMindRequest::Whoami), "{request:?}"); @@ -625,6 +740,33 @@ mod tests { assert_eq!((page.matched, page.as_of, page.scores), (0, 7, None)); } + /// A chat search is refused by field at the door, and the published + /// response marks every citation required. + #[test] + fn a_chat_query_is_refused_by_field_and_a_hit_cites_room_ids_and_times() { + let ok = ChatQuery { texts: vec![], rooms: vec![Short("!a:b".into()), Short("#c:b".into())], after_ms: Some(1), before_ms: Some(1), top_k: 50 }; + assert!(ok.check().is_ok()); + let field = |query: ChatQuery| match query.check() { + Err(MindRefusal::SelectionInvalid { field, .. }) => field, + other => panic!("{other:?}"), + }; + assert_eq!(field(ChatQuery { rooms: vec![Short("swarm".into())], ..ok.clone() }), "rooms"); + assert_eq!(field(ChatQuery { rooms: vec![Short("#".into())], ..ok.clone() }), "rooms"); + assert_eq!(field(ChatQuery { before_ms: Some(0), ..ok.clone() }), "before_ms"); + assert_eq!(field(ChatQuery { top_k: 0, ..ok.clone() }), "top_k"); + assert_eq!(field(ChatQuery { top_k: 51, ..ok.clone() }), "top_k"); + assert_eq!(field(ChatQuery { texts: vec![Line(" ".into())], ..ok.clone() }), "texts"); + assert_eq!(field(ChatQuery { texts: vec![Line("x".into()); 9], ..ok.clone() }), "texts"); + assert_eq!(field(ChatQuery { rooms: vec![Short("!a:b".into()); 17], ..ok }), "rooms"); + let response = serde_json::to_value(schemars::schema_for!(HuginnMindResponse)).unwrap(); + let required: Vec<&str> = response["$defs"]["ChatHit"]["required"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect(); + for field in ["room_id", "first_event_id", "last_event_id", "ts_start_ms", "ts_end_ms", "messages"] { + assert!(required.contains(&field), "{field} must be required"); + } + let message: Vec<&str> = response["$defs"]["ChatMessage"]["required"].as_array().unwrap().iter().filter_map(|v| v.as_str()).collect(); + assert_eq!(message, ["event_id", "sender", "ts_ms", "body"]); + } + /// The two files the catalog publishes are the derivation of the two /// types, byte for byte. #[test] diff --git a/schemas/cultnet/huginn.mind_request.v1.schema.json b/schemas/cultnet/huginn.mind_request.v1.schema.json index 5c3ea7a..233da8d 100644 --- a/schemas/cultnet/huginn.mind_request.v1.schema.json +++ b/schemas/cultnet/huginn.mind_request.v1.schema.json @@ -158,6 +158,31 @@ "SearchSources" ] }, + { + "description": "Search of the chat index: chunks of the rooms the Minds are meant to\nhave, by room, by time span and by meaning. Read-only, like\n`SearchSources`, and queued with it.", + "type": "object", + "properties": { + "SearchChat": { + "type": "object", + "properties": { + "instance": { + "$ref": "#/$defs/Slug" + }, + "query": { + "$ref": "#/$defs/ChatQuery" + } + }, + "required": [ + "instance", + "query" + ] + } + }, + "additionalProperties": false, + "required": [ + "SearchChat" + ] + }, { "description": "What others did in `repos` on the forge since the `after` cursor: the\ndaemon reads the forge and no document of the mind, and answers\n`Refused(Unavailable)` when it has no forge or the forge does not answer\nin time. The repos are the asker's to name (her stewardships); the\ninstance's own login is dropped from the answer.", "type": "object", @@ -272,6 +297,50 @@ "deletion_line" ] }, + "ChatQuery": { + "description": "A chat search. The selectors compose as AND. `rooms` are any-of: a value\nbeginning `!` is a room id, `#` a canonical alias. `after_ms` and\n`before_ms` (Unix milliseconds) keep the chunks whose span overlaps\n[after, before]. `texts` (none to `SOURCE_TEXTS_MAX`) is the meaning match,\nseveral fused as in a source search; with none the search is a listing:\nchunks oldest first, at most `top_k`, `ChatHits.truncated` saying more\nexist, and the caller narrows with `after_ms` = the last hit's\n`ts_end_ms + 1`.", + "type": "object", + "properties": { + "after_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "before_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0 + }, + "rooms": { + "type": "array", + "items": { + "$ref": "#/$defs/Short" + } + }, + "texts": { + "type": "array", + "items": { + "$ref": "#/$defs/Line" + } + }, + "top_k": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "texts", + "rooms", + "top_k" + ] + }, "ClaimOutcome": { "type": "string", "enum": [ diff --git a/schemas/cultnet/huginn.mind_response.v1.schema.json b/schemas/cultnet/huginn.mind_response.v1.schema.json index 70cbdf1..4adbc43 100644 --- a/schemas/cultnet/huginn.mind_response.v1.schema.json +++ b/schemas/cultnet/huginn.mind_response.v1.schema.json @@ -113,6 +113,18 @@ "Sources" ] }, + { + "type": "object", + "properties": { + "Chat": { + "$ref": "#/$defs/ChatHits" + } + }, + "additionalProperties": false, + "required": [ + "Chat" + ] + }, { "type": "object", "properties": { @@ -236,6 +248,110 @@ "deletion_line" ] }, + "ChatHit": { + "description": "One chunk of consecutive messages in one room. Every citation is\nrequired: only the room's alias may be absent, because a room may have\nnone. The span and the first and last ids are those of `messages` as\nreturned, after any trimming to the asked span.", + "type": "object", + "properties": { + "first_event_id": { + "type": "string" + }, + "last_event_id": { + "type": "string" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatMessage" + } + }, + "room_alias": { + "type": [ + "string", + "null" + ] + }, + "room_id": { + "type": "string" + }, + "score": { + "description": "Cosine similarity for one text, the reciprocal-rank sum for several,\nnone in a listing.", + "type": [ + "number", + "null" + ], + "format": "float" + }, + "ts_end_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "ts_start_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "room_id", + "first_event_id", + "last_event_id", + "ts_start_ms", + "ts_end_ms", + "messages" + ] + }, + "ChatHits": { + "description": "The answer to a chat search: oldest first in a listing, nearest first with\ntexts.", + "type": "object", + "properties": { + "hits": { + "type": "array", + "items": { + "$ref": "#/$defs/ChatHit" + } + }, + "indexed_under": { + "$ref": "#/$defs/SourceModel" + }, + "truncated": { + "description": "A listing had more chunks than `top_k`.", + "type": "boolean" + } + }, + "required": [ + "indexed_under", + "truncated", + "hits" + ] + }, + "ChatMessage": { + "description": "One message of a chunk, as the room held it.", + "type": "object", + "properties": { + "body": { + "type": "string" + }, + "event_id": { + "type": "string" + }, + "sender": { + "description": "The sender's Matrix id.", + "type": "string" + }, + "ts_ms": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "event_id", + "sender", + "ts_ms", + "body" + ] + }, "CitationRole": { "description": "The referring field of a citation edge, one per leaf field that names\nanother document. The wire spelling is the field's own name, so a client\nreads an edge's role as the field it came from.", "type": "string",