From 7636737d60565e33a31f9e3ea27db245a9dd74da Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 07:29:17 +0200 Subject: [PATCH 001/520] BP-3 1: an oversize answer is deferred to the body plane deliver replaces within_window as the one owner of how an answer leaves: whole when the envelope fits the window, deferred when the payload is within MAX_DEFERRED_BODY_BYTES (64 MiB), refused by name past it. A deferred answer is HuginnMindResponse::Deferred(DeferredAnswer { manifest }), the reference's CultMeshCdnArtifactManifest; the chunks are asked for on the session the client holds and answered by cultnet-rs's answer_content_chunk_request. The body is held by DeferredBodies (256 MiB budget, sliding 60 s TTL, least-recently- touched eviction, shared chunks stored once), in memory only. The mind knows nothing of deferral. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/bodies.rs | 270 +++++++ crates/huginn-daemon/src/lib.rs | 5 +- crates/huginn-daemon/src/serve.rs | 753 ++++++++++-------- crates/huginn-mind/src/lib.rs | 2 +- crates/huginn-mind/src/refusal.rs | 17 +- crates/huginn-mind/src/wire.rs | 35 +- .../huginn.mind_response.v1.schema.json | 28 +- 7 files changed, 784 insertions(+), 326 deletions(-) create mode 100644 crates/huginn-daemon/src/bodies.rs diff --git a/crates/huginn-daemon/src/bodies.rs b/crates/huginn-daemon/src/bodies.rs new file mode 100644 index 0000000..1ec113e --- /dev/null +++ b/crates/huginn-daemon/src/bodies.rs @@ -0,0 +1,270 @@ +//! The deferred bodies: answers too large for one send, held for the client to +//! fetch chunk by chunk. Ephemeral transport state, never a document store and +//! never world truth: it is lost when the process exits, and a client that +//! finds a body gone asks the operation again. +//! +//! Identity is the manifest's `contentHash`, so the same bytes deferred twice +//! are one body. Chunking, hashing and the manifest are `cultnet-rs`'s; what is +//! owned here is only how long and how much is kept. +//! +//! A body expires when it has been untouched for the TTL, where retaining it +//! again and serving any of its chunks both touch it. When the stored bytes +//! exceed the budget the body touched longest ago is evicted first. A chunk +//! that two bodies share is stored once and lives until the last of them goes. +//! Time is an argument, so nothing here reads a clock. + +use std::cell::Cell; +use std::collections::{BTreeSet, HashMap}; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use cultnet_rs::{CultMeshCdnArtifactManifest, CultMeshCdnChunk}; + +/// The size of one chunk of a deferred body. Four fit one send with room for a +/// control-plane answer beside them on the same session. +pub const DEFERRED_CHUNK_BYTES: usize = 256 * 1024; +/// The largest answer that is deferred rather than refused, measured on the +/// encoded payload. +pub const MAX_DEFERRED_BODY_BYTES: u64 = 64 * 1024 * 1024; +/// What the daemon keeps across all deferred bodies. +pub const DEFERRED_BUDGET_BYTES: u64 = 256 * 1024 * 1024; + +struct Chunk { + bytes: Vec, + /// The content hashes of the retained bodies that reference this chunk. + owners: BTreeSet, +} + +struct Body { + chunk_hashes: BTreeSet, + touched_at: Cell>, + /// Orders touches when the clock does not: the largest is the newest. + touch_order: Cell, +} + +pub struct DeferredBodies { + budget: u64, + ttl: chrono::Duration, + chunks: HashMap, + bodies: HashMap, + stored: u64, + touches: Cell, +} + +impl DeferredBodies { + pub fn new(budget: u64, ttl: Duration) -> Self { + Self { + budget, + ttl: chrono::Duration::from_std(ttl).unwrap_or(chrono::Duration::MAX), + chunks: HashMap::new(), + bodies: HashMap::new(), + stored: 0, + touches: Cell::new(0), + } + } + + /// Bytes held: each distinct chunk once. + #[cfg(test)] + pub fn stored_bytes(&self) -> u64 { + self.stored + } + + #[cfg(test)] + pub fn body_count(&self) -> usize { + self.bodies.len() + } + + /// Holds a body, or, when its bytes are already held, touches it and + /// stores nothing. Then evicts, least recently touched first, while the + /// stored bytes exceed the budget and another body could be evicted + /// instead: the body just retained is the last to go, and one that alone + /// exceeds the budget is kept whole. + pub fn retain(&mut self, manifest: &CultMeshCdnArtifactManifest, chunks: Vec, now: DateTime) { + let content_hash = manifest.content_hash.clone(); + if let Some(body) = self.bodies.get(&content_hash) { + self.touch(body, now); + return; + } + let mut chunk_hashes = BTreeSet::new(); + for chunk in chunks { + chunk_hashes.insert(chunk.chunk_hash.clone()); + let held = self.chunks.entry(chunk.chunk_hash).or_insert_with(|| { + self.stored += chunk.payload.len() as u64; + Chunk { bytes: chunk.payload, owners: BTreeSet::new() } + }); + held.owners.insert(content_hash.clone()); + } + let body = Body { chunk_hashes, touched_at: Cell::new(now), touch_order: Cell::new(0) }; + self.touch(&body, now); + self.bodies.insert(content_hash, body); + while self.stored > self.budget && self.bodies.len() > 1 { + let oldest = self + .bodies + .iter() + .min_by_key(|(_, body)| body.touch_order.get()) + .map(|(hash, _)| hash.clone()) + .expect("more than one body is held"); + self.drop_body(&oldest); + } + } + + /// One chunk's bytes by its normalised hash, touching every body that + /// references it. Serving a chunk is what keeps its body alive. + pub fn chunk(&self, hash: &str, now: DateTime) -> Option<&[u8]> { + let chunk = self.chunks.get(hash)?; + for owner in &chunk.owners { + self.touch(&self.bodies[owner], now); + } + Some(&chunk.bytes) + } + + /// Drops every body untouched for the TTL as of `now`. A clock that reads + /// earlier than a touch expires nothing. + pub fn expire(&mut self, now: DateTime) { + let expired: Vec = self + .bodies + .iter() + .filter(|(_, body)| now.signed_duration_since(body.touched_at.get()) >= self.ttl) + .map(|(hash, _)| hash.clone()) + .collect(); + for hash in expired { + self.drop_body(&hash); + } + } + + fn touch(&self, body: &Body, now: DateTime) { + body.touched_at.set(now); + self.touches.set(self.touches.get() + 1); + body.touch_order.set(self.touches.get()); + } + + fn drop_body(&mut self, content_hash: &str) { + let Some(body) = self.bodies.remove(content_hash) else { return }; + for hash in body.chunk_hashes { + let chunk = self.chunks.get_mut(&hash).expect("a retained body's chunks are held"); + chunk.owners.remove(content_hash); + if chunk.owners.is_empty() { + self.stored -= chunk.bytes.len() as u64; + self.chunks.remove(&hash); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use cultnet_rs::pack_content; + + fn at(seconds: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + seconds, 0).unwrap() + } + + /// A body of `pieces` chunks of 4 bytes each, distinct per `tag`. + fn body(tag: u8, pieces: usize) -> (CultMeshCdnArtifactManifest, Vec) { + let mut bytes = Vec::new(); + for piece in 0..pieces { + bytes.extend_from_slice(&[tag, piece as u8, 7, 7]); + } + pack_content("t", "package", "", "", "", &bytes, 4).unwrap() + } + + fn held(store: &DeferredBodies, manifest: &CultMeshCdnArtifactManifest, now: DateTime) -> bool { + manifest.chunks.iter().all(|chunk| store.chunk(&chunk.chunk_hash, now).is_some()) + } + + const TTL: Duration = Duration::from_secs(60); + + /// The same bytes deferred twice are one body, counted once; serving a + /// chunk slides the expiry; an untouched body is gone exactly at the TTL. + #[test] + fn retention_is_idempotent_and_sliding() { + let mut store = DeferredBodies::new(1_000, TTL); + let (manifest, chunks) = body(1, 3); + store.retain(&manifest, chunks.clone(), at(0)); + store.retain(&manifest, chunks, at(1)); + assert_eq!((store.body_count(), store.stored_bytes()), (1, 12), "one body, its bytes counted once"); + + // Retaining again at t=1 touched it, so it lives to t=61, not t=60. + store.expire(at(60)); + assert_eq!(store.body_count(), 1, "the second retain refreshed it"); + // Serving a chunk at t=60 touches it, so t=119 is inside a fresh TTL. + assert!(store.chunk(&manifest.chunks[0].chunk_hash, at(60)).is_some()); + store.expire(at(119)); + assert_eq!(store.body_count(), 1, "serving a chunk slid the expiry"); + store.expire(at(120)); + assert_eq!((store.body_count(), store.stored_bytes()), (0, 0), "untouched for the TTL is gone"); + + // An untouched body is gone at the TTL and not before it. + let (other, chunks) = body(2, 1); + store.retain(&other, chunks, at(200)); + store.expire(at(259)); + assert_eq!(store.body_count(), 1); + store.expire(at(260)); + assert_eq!(store.body_count(), 0); + } + + /// Time going backwards expires nothing. + #[test] + fn a_clock_behind_the_last_touch_expires_nothing() { + let mut store = DeferredBodies::new(1_000, TTL); + let (manifest, chunks) = body(1, 1); + store.retain(&manifest, chunks, at(100)); + store.expire(at(0)); + assert_eq!(store.body_count(), 1); + } + + /// Over the budget the body touched longest ago goes first, not the + /// newest and not the first inserted, and a chunk two bodies share stays + /// while either lives. + #[test] + fn eviction_is_least_recently_touched_and_bounded_in_bytes() { + // Budget 12 bytes: three chunks of 4. + let mut store = DeferredBodies::new(12, TTL); + let (a, chunks_a) = body(1, 1); + let (b, chunks_b) = body(2, 1); + let (c, chunks_c) = body(3, 1); + store.retain(&a, chunks_a, at(0)); + store.retain(&b, chunks_b, at(1)); + // The oldest-inserted body is touched last, so insertion order and + // recency disagree about who goes. + assert!(store.chunk(&a.chunks[0].chunk_hash, at(2)).is_some()); + store.retain(&c, chunks_c, at(3)); + assert_eq!(store.stored_bytes(), 12); + let (d, chunks_d) = body(4, 1); + store.retain(&d, chunks_d, at(4)); + assert_eq!(store.stored_bytes(), 12, "bounded by the budget"); + assert!(!held(&store, &b, at(5)), "the least recently touched went"); + assert!(held(&store, &a, at(5)) && held(&store, &c, at(5)) && held(&store, &d, at(5))); + + // A shared chunk survives its first owner. + let mut store = DeferredBodies::new(12, TTL); + let (x, chunks_x) = body(5, 2); // chunks x0 x1 + store.retain(&x, chunks_x, at(0)); + let (y, mut chunks_y) = body(6, 1); // chunk y0, plus x1 shared + let mut manifest_y = y.clone(); + manifest_y.content_hash = "shared-owner".into(); + chunks_y.push(CultMeshCdnChunk { + chunk_hash: x.chunks[1].chunk_hash.clone(), + payload: vec![5, 1, 7, 7], + }); + store.retain(&manifest_y, chunks_y, at(1)); + assert_eq!(store.stored_bytes(), 12, "the shared chunk is stored once"); + // Evict x (older): its private chunk goes, the shared one stays. + let (z, chunks_z) = body(7, 1); + store.retain(&z, chunks_z, at(2)); + assert_eq!(store.body_count(), 2, "x was evicted"); + assert!(store.chunk(&x.chunks[0].chunk_hash, at(3)).is_none(), "x's private chunk is gone"); + assert!(store.chunk(&x.chunks[1].chunk_hash, at(3)).is_some(), "the shared chunk lives with y"); + } + + /// A body that alone exceeds the budget is kept whole rather than + /// evicting itself, so a manifest handed out is never a lie. + #[test] + fn a_body_over_the_budget_by_itself_is_kept() { + let mut store = DeferredBodies::new(4, TTL); + let (manifest, chunks) = body(1, 3); + store.retain(&manifest, chunks, at(0)); + assert!(held(&store, &manifest, at(0))); + } +} diff --git a/crates/huginn-daemon/src/lib.rs b/crates/huginn-daemon/src/lib.rs index ee36d7c..d2dc669 100644 --- a/crates/huginn-daemon/src/lib.rs +++ b/crates/huginn-daemon/src/lib.rs @@ -6,12 +6,15 @@ //! request": it dispatches, it asks `Mind::require_instance` for every read //! that names an instance, and it compares nothing itself. `envelope` owns "is //! this a request at all": a message it cannot decode is answered with a typed -//! failure and touches no mind. `serve` owns "which session gets which reply". +//! failure and touches no mind. `serve` owns "which session gets which reply" +//! and whether an answer is delivered whole, deferred or refused; `bodies` +//! owns how long and how much of a deferred answer is held. //! //! Every admission rule, the instance check, the receipt, the derived status //! and the wire vocabulary are `huginn-mind`'s. The leaf and the store type //! are reached through `huginn_mind`, so one crate pins one revision of each. +pub mod bodies; pub mod daemon; pub mod envelope; pub mod serve; diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index 888d5a2..08698a3 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -7,14 +7,23 @@ //! spins with logging rather than exiting, and Cut 14's health check is the //! observer. //! -//! One answer may be `MAX_RESPONSE_BYTES` encoded, 1,228,800 today: the +//! One send carries `MAX_RESPONSE_BYTES` encoded, 1,228,800 today: the //! reliable window one session holds, `MAX_PENDING_RELIABLE_PACKETS` packets -//! of `MAX_FRAGMENT_BYTES`. A larger answer is refused by name, -//! `MindRefusal::ResponseTooLarge`, carrying the encoded size and the limit, -//! and the caller narrows its own request. This is the current bound and not a -//! design target: a single cut spec at the leaf's own bounds does not fit it, -//! and what a list read should return is an open question, so a client author -//! should expect the number to move. +//! of `MAX_FRAGMENT_BYTES`. `deliver` decides every answer's plane: one that +//! fits goes whole; a larger one whose encoded payload is at most +//! `MAX_DEFERRED_BODY_BYTES` is held by `DeferredBodies` and answered with +//! `HuginnMindResponse::Deferred`, a `CultMeshCdnArtifactManifest`, and the +//! client fetches the payload with `cultnet_rs::fetch_content`, one +//! `cultmesh.content_chunk_request.v1` at a time on the session it already +//! holds, exactly as the C# reference's `CultMeshLegacyRudpContentServer` is +//! asked. The fetched bytes are the named MessagePack of the +//! `HuginnMindResponse` the mind gave. A body the daemon will not defer is +//! refused by name, `MindRefusal::ResponseTooLarge`, carrying the payload size +//! and `MAX_DEFERRED_BODY_BYTES`, and the caller narrows its own request. A +//! deferred body is memory only: it expires untouched after +//! `ServeOptions::deferred_ttl`, is evicted past `DEFERRED_BUDGET_BYTES`, and +//! is gone when the process exits, so a chunk request that finds it gone is +//! answered `found: false` and the client asks the operation again. //! //! Four things this cut does not do, which a client has nowhere else to learn: //! @@ -50,31 +59,43 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use anyhow::{Context, Result, ensure}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; use chrono::{DateTime, Utc}; use cultnet_rs::{ CULTNET_OPERATION_CONNECTION_ID, CultNetMessage, CultNetRudpServerEvent, CultNetRudpServerHub, CultNetRudpServerHubOptions, CultNetSchemaKind, CultNetSchemaRegistration, CultNetSchemaRegistry, - CultNetWireContract, decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, + CultNetWireContract, answer_content_chunk_request, decode_cultnet_message_from_slice, + encode_cultnet_message_to_vec, pack_content, }; use huginn_mind::epiphany_pipeline::Slug; use huginn_mind::wire::{ - HuginnMindResponse, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON, MIND_RESPONSE_SCHEMA, + DeferredAnswer, HuginnMindResponse, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON, MIND_RESPONSE_SCHEMA, MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, }; use huginn_mind::{MindRefusal, MindStore, OwnedRedbMessagePackBackingStore}; +use crate::bodies::{ + DEFERRED_BUDGET_BYTES, DEFERRED_CHUNK_BYTES, DeferredBodies, MAX_DEFERRED_BODY_BYTES, +}; use crate::daemon::{Daemon, IndexSink, NoIndex}; -use crate::envelope::{decode_request, encode_failure, encode_response}; +use crate::envelope::{OperationFailure, decode_request, encode_failure, encode_response}; -/// What the loop does when nothing is waiting. +/// What the loop does when nothing is waiting. A deferred body untouched for +/// `deferred_ttl`, twice the session timeout, is dropped. pub struct ServeOptions { pub session_timeout: Duration, pub idle_sleep: Duration, + pub deferred_ttl: Duration, } impl Default for ServeOptions { fn default() -> Self { - Self { session_timeout: Duration::from_secs(30), idle_sleep: Duration::from_millis(2) } + Self { + session_timeout: Duration::from_secs(30), + idle_sleep: Duration::from_millis(2), + deferred_ttl: Duration::from_secs(60), + } } } @@ -164,20 +185,24 @@ pub fn schema_registry() -> Result { /// One message in, one message out. Nothing here reaches a mind except through /// `Daemon::handle`, and an envelope that does not decode never gets that far. +/// A chunk request is answered from `bodies` alone, on the session it came in +/// on; every operation's reply leaves through `deliver`. pub fn answer>( daemon: &mut Daemon, registry: &CultNetSchemaRegistry, + bodies: &mut DeferredBodies, message: CultNetMessage, now: DateTime, ) -> CultNetMessage { let runtime_id = daemon.runtime_id(); + bodies.expire(now); match &message { CultNetMessage::OperationRequest { message_id, operation, .. } => match decode_request(&message) { Ok((message_id, request)) => { let operation = request.operation(); let response = daemon.handle(request, now); let reply = encode_or_fail(&message_id, operation, &response, &runtime_id); - within_window(reply, &message_id, operation, &runtime_id) + deliver(reply, &message_id, operation, &runtime_id, bodies, now) } Err(failure) => encode_failure(message_id, operation, &failure, &runtime_id), }, @@ -185,9 +210,13 @@ pub fn answer>( Ok(response) => response, Err(error) => CultNetMessage::Error { error: format!("{error:#}"), code: None, details: None }, }, + CultNetMessage::ContentChunkRequest { .. } => { + answer_content_chunk_request(&message, |hash| bodies.chunk(hash, now)) + } _ => CultNetMessage::Error { error: format!( - "{MIND_SERVICE_ID} answers cultnet.operation_request.v0 and cultnet.schema_catalog_request.v0" + "{MIND_SERVICE_ID} answers cultnet.operation_request.v0, cultnet.schema_catalog_request.v0 \ + and cultmesh.content_chunk_request.v1" ), code: None, details: None, @@ -216,40 +245,73 @@ fn encode_or_fail( } } -/// The reply, or a typed refusal saying it will not fit. The hub cuts a reply -/// into `MAX_FRAGMENT_BYTES` packets and refuses a send needing more than -/// `MAX_PENDING_RELIABLE_PACKETS` of them; that refusal is a transport error -/// the client never sees, so it waits for an answer nothing will send. The -/// size is therefore measured here, against the window this module configured, -/// before a send that can already be seen to fail is attempted, and the client -/// is told by name how large the answer was and how large one may be. +/// How one answer leaves: whole, deferred or refused, in that order, and +/// nothing else decides. The hub cuts a reply into `MAX_FRAGMENT_BYTES` packets +/// and refuses a send needing more than `MAX_PENDING_RELIABLE_PACKETS` of them; +/// that refusal is a transport error the client never sees, so it waits for an +/// answer nothing will send. The envelope is therefore measured here, against +/// the window this module configured, before a send that can already be seen to +/// fail is attempted. /// -/// Nothing is truncated, paginated or retried: the caller narrows its own -/// request. The refusal rides the response schema like every other refusal, so -/// it is not a second thing for a client to parse. -fn within_window(reply: CultNetMessage, message_id: &str, operation: &str, runtime_id: &str) -> CultNetMessage { +/// A reply that fits is returned as it is. A larger one whose payload, the +/// named MessagePack of the response, is at most `MAX_DEFERRED_BODY_BYTES` is +/// packed by CultNet's content plane, retained in `bodies`, and answered with +/// `Deferred` and the manifest. That answer is a small fixed-shape envelope and +/// is never itself measured or deferred. A larger payload is refused by name +/// with its own size and the deferral bound: nothing is truncated or paginated, +/// and the caller narrows its own request. Both answers ride the response +/// schema like every other, so a client parses no second vocabulary. +fn deliver( + reply: CultNetMessage, + message_id: &str, + operation: &str, + runtime_id: &str, + bodies: &mut DeferredBodies, + now: DateTime, +) -> CultNetMessage { + let not_encodable = |message: String| { + encode_failure( + message_id, + operation, + &OperationFailure { code: "response-not-encodable".into(), message }, + runtime_id, + ) + }; let encoded = match encode_cultnet_message_to_vec(&reply, CultNetWireContract::CultNetSchemaV0) { Ok(bytes) => bytes.len() as u64, - Err(error) => { - return encode_failure( - message_id, - operation, - &crate::envelope::OperationFailure { - code: "response-not-encodable".into(), - message: format!("{error:#}"), - }, - runtime_id, - ); - } + Err(error) => return not_encodable(format!("{error:#}")), }; if encoded <= MAX_RESPONSE_BYTES { return reply; } - let refusal = HuginnMindResponse::Refused(MindRefusal::ResponseTooLarge { - bytes: encoded, - limit: MAX_RESPONSE_BYTES, - }); - encode_or_fail(message_id, operation, &refusal, runtime_id) + let payload = match &reply { + CultNetMessage::OperationResponse { payload, .. } => STANDARD.decode(payload), + _ => return not_encodable("only an operation response is deferred".into()), + }; + let payload = match payload { + Ok(bytes) => bytes, + Err(error) => return not_encodable(format!("the payload is not base64: {error}")), + }; + let size = payload.len() as u64; + if size > MAX_DEFERRED_BODY_BYTES { + let refusal = HuginnMindResponse::Refused(MindRefusal::ResponseTooLarge { + bytes: size, + limit: MAX_DEFERRED_BODY_BYTES, + }); + return encode_or_fail(message_id, operation, &refusal, runtime_id); + } + let (manifest, chunks) = pack_content( + "huginn.mind_response", + "package", + "", + "application/vnd.gamecult.huginn.mind-response+msgpack", + &now.to_rfc3339(), + &payload, + DEFERRED_CHUNK_BYTES, + ) + .expect("the deferral constants are a valid pack"); + bodies.retain(&manifest, chunks, now); + encode_or_fail(message_id, operation, &HuginnMindResponse::Deferred(DeferredAnswer { manifest }), runtime_id) } /// Until `stopping`: expire what timed out, resend what was not acknowledged, @@ -264,6 +326,7 @@ pub fn run>( options: &ServeOptions, ) -> Result<()> { let timeout_ms = options.session_timeout.as_millis() as u64; + let mut bodies = DeferredBodies::new(DEFERRED_BUDGET_BYTES, options.deferred_ttl); while !stopping.load(Ordering::Relaxed) { hub.remove_timed_out_sessions(timeout_ms); if let Err(error) = hub.poll_resends() { @@ -293,7 +356,7 @@ pub fn run>( continue; } }; - let reply = answer(daemon, registry, message, Utc::now()); + let reply = answer(daemon, registry, &mut bodies, message, Utc::now()); if let Err(error) = hub.send_schema_message(&session, &reply) { eprintln!("huginn: {} did not receive its reply: {error:#}", session.remote_addr); } @@ -314,9 +377,7 @@ mod tests { }; use huginn_mind::epiphany_pipeline::{PipelineKind, PipelineRef}; use crate::envelope::{FAILURE_SCHEMA, decode_response, encode_request}; - use base64::Engine; - use base64::engine::general_purpose::STANDARD; - use cultnet_rs::{CultMesh, CultMeshRudpSocketOptions}; + use cultnet_rs::{CultMesh, CultMeshCdnArtifactManifest, CultMeshRudpSocketOptions, fetch_content}; use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, MindStatus}; use cultnet_rs::{FieldPredicate, Selection}; use huginn_mind::{MindRefusal, PipelineAdmissionOutcome, PipelinePageItems}; @@ -391,14 +452,14 @@ mod tests { ("m-5b", request("m-5b", MIND_SERVICE_ID, "Admit", MIND_REQUEST_SCHEMA, admit), "operation-mismatch"), ]; for (id, message, code) in cases { - let reply = answer(&mut daemon, ®istry, message, now()); + let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), message, now()); assert_eq!(rejected(&reply).0, id, "the client's own correlation key is echoed"); assert_eq!(rejected(&reply).1, code); } // The sixth code, `not-an-operation-request`, is the codec's and is // pinned in `envelope::tests`: `answer` never produces it, because a // message of another family is answered with `Error` instead. - let reply = answer(&mut daemon, ®istry, CultNetMessage::Error { error: "hello".into(), code: None, details: None }, now()); + let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), CultNetMessage::Error { error: "hello".into(), code: None, details: None }, now()); let CultNetMessage::Error { error, .. } = &reply else { panic!("expected an error, got {reply:?}") }; assert!( !error.is_empty() && error.contains("cultnet.operation_request.v0") && error.contains("cultnet.schema_catalog_request.v0"), @@ -410,7 +471,7 @@ mod tests { // A mind's refusal is an answer on the response schema, not a failure // of the envelope: the client decodes it as the typed refusal it is. let read = HuginnMindRequest::Query { instance: slug(OTHER), selection: Selection::default(), semantic: None }; - let reply = answer(&mut daemon, ®istry, encode_request("m-r", &read, None).unwrap(), now()); + let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), encode_request("m-r", &read, None).unwrap(), now()); let CultNetMessage::OperationResponse { status, payload_schema, .. } = &reply else { panic!("expected an operation response, got {reply:?}"); }; @@ -428,7 +489,7 @@ mod tests { schema_ids: None, kinds: None, }; - let reply = answer(&mut daemon, ®istry, catalog, now()); + let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), catalog, now()); let CultNetMessage::SchemaCatalogResponse { message_id, schemas } = reply else { panic!("expected a catalog response"); }; @@ -532,29 +593,109 @@ mod tests { assert_eq!(documents(&mut daemon), 1, "the admission that crossed the wire landed"); } - /// An answer larger than one send can carry is a refusal by name, not a - /// transport failure the client waits out. The wide document is one cut spec - /// with every list the leaf bounds filled: about a megabyte of field - /// content, which base64 in the operation envelope carries past the window - /// this module configures, so a single read of it is already over. The - /// fitting one is the same document narrowed until its answer lands inside - /// the window, and it is delivered whole. + fn encoded_len(message: &CultNetMessage) -> u64 { + encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap().len() as u64 + } + + const TTL: Duration = Duration::from_secs(60); + + fn fresh_bodies() -> DeferredBodies { + DeferredBodies::new(DEFERRED_BUDGET_BYTES, TTL) + } + + type Client = cultnet_rs::CultNetRudpSocketTransportConnection; + type BodyDaemon = Daemon; + + /// One request on the client's own session and its reply, the way + /// `fetch_content` is asked for each chunk. + fn exchange(client: &mut Client, message: &CultNetMessage) -> Result { + client.send_schema_message(message)?; + for _ in 0..3000 { + client.poll_resends()?; + if let Some(reply) = client.receive_schema_message_once()? { + return Ok(reply); + } + std::thread::sleep(Duration::from_millis(2)); + } + anyhow::bail!("no reply") + } + + fn deferred_manifest(reply: &CultNetMessage) -> CultMeshCdnArtifactManifest { + match decode_response(reply).unwrap().1.expect("an answer, not an envelope failure") { + HuginnMindResponse::Deferred(DeferredAnswer { manifest }) => manifest, + other => panic!("expected a deferred answer, got {other:?}"), + } + } + + fn chunk_request(manifest: &CultMeshCdnArtifactManifest, index: usize) -> CultNetMessage { + let chunk = &manifest.chunks[index]; + CultNetMessage::ContentChunkRequest { + message_id: format!("c-{index}"), + chunk_hash: chunk.chunk_hash.clone(), + record_key: chunk.record_key.clone(), + expected_size_bytes: chunk.size_bytes, + } + } + + fn found(reply: &CultNetMessage) -> bool { + let CultNetMessage::ContentChunkResponse { found, .. } = reply else { + panic!("expected a content chunk response, got {reply:?}"); + }; + *found + } + + fn document_query() -> HuginnMindRequest { + HuginnMindRequest::Query { + instance: slug(INSTANCE), + selection: Selection { projection: "document".into(), ..Selection::default() }, + semantic: None, + } + } + + /// One operation asked of `answer` at an injected time, and the manifest of + /// the deferred answer it gives back. + fn ask_deferred( + daemon: &mut BodyDaemon, + bodies: &mut DeferredBodies, + message_id: &str, + read: &HuginnMindRequest, + at: DateTime, + ) -> CultMeshCdnArtifactManifest { + let registry = schema_registry().unwrap(); + deferred_manifest(&answer(daemon, ®istry, bodies, encode_request(message_id, read, None).unwrap(), at)) + } + + /// Whether the first chunk of a manifest is still served at an injected time. + fn first_chunk_found( + daemon: &mut BodyDaemon, + bodies: &mut DeferredBodies, + manifest: &CultMeshCdnArtifactManifest, + at: DateTime, + ) -> bool { + let registry = schema_registry().unwrap(); + found(&answer(daemon, ®istry, bodies, chunk_request(manifest, 0), at)) + } + + /// An answer larger than one send can carry is deferred to the body plane + /// and fetches, on the same session, to the answer the mind gave. The wide + /// document is one cut spec with every list the leaf bounds filled: about a + /// megabyte of field content, which base64 in the operation envelope + /// carries past the window this module configures, so a single read of it is + /// already over. The fitting one is the same document narrowed until its + /// envelope lands inside the window, and it is delivered whole. /// - /// Both halves are load-bearing. Without the refused reads a daemon that - /// never measured would pass; without the delivered one a daemon that - /// refused everything, or configured a window smaller than the number it - /// measures against, would pass too. The delivered answer sits within one - /// packet-count's slack of the limit, so shrinking the configured window - /// breaks it. + /// The client holds the `Deferred` answer before it fetches, so the size + /// the daemon decided on is observed at the decision and not only + /// downstream of the fetch. #[test] - fn an_answer_too_large_for_one_send_is_a_typed_refusal_that_reaches_the_client() { + fn an_oversize_answer_arrives_deferred_and_fetches_to_the_same_answer() { let (_root, mut daemon) = seeded_wide(); let registry = schema_registry().unwrap(); - let sized = |daemon: &mut Daemon, id: PipelineRef| { + let sized = |daemon: &mut BodyDaemon, id: PipelineRef| { let view = daemon.handle(HuginnMindRequest::View { instance: slug(INSTANCE), id }, now()); let message = encode_response("m-0", "view", &view, "huginn-yggdrasil").unwrap(); - encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).unwrap().len() as u64 + encoded_len(&message) }; let wide = spec_ref(&mut daemon, WIDE_CUT); let fitting = spec_ref(&mut daemon, FITTING_CUT); @@ -567,6 +708,13 @@ mod tests { "the delivered answer must sit close enough to the limit that the window's value is load-bearing" ); + let reads = [ + ("m-q", document_query()), + ("m-v", HuginnMindRequest::View { instance: slug(INSTANCE), id: wide }), + ]; + let expected: Vec = + reads.iter().map(|(_, read)| daemon.handle(read.clone(), now())).collect(); + let mut hub = bind("127.0.0.1:0".parse().unwrap(), &daemon.runtime_id()).unwrap(); let endpoint = format!("rudp://{}", hub.local_addr().unwrap()); let mut client = CultMesh::create_rudp_client_for_endpoint( @@ -587,20 +735,6 @@ mod tests { } assert!(client.connected()); - let reads = [ - // The whole documents: a header page is small by construction, so - // the transport's bound is measured on the projection that can - // exceed it. - ( - "m-q", - HuginnMindRequest::Query { - instance: slug(INSTANCE), - selection: Selection { projection: "document".into(), ..Selection::default() }, - semantic: None, - }, - ), - ("m-v", HuginnMindRequest::View { instance: slug(INSTANCE), id: wide }), - ]; let stopping = Arc::new(AtomicBool::new(false)); let loop_stopping = Arc::clone(&stopping); let serving = std::thread::spawn(move || { @@ -609,50 +743,30 @@ mod tests { }); // One read at a time: the window is per session and counts what is - // still unacknowledged, so two large answers in flight together are a - // separate limit this gate does not measure. - for (message_id, request) in reads { - client.send_schema_message(&encode_request(message_id, &request, None).unwrap()).unwrap(); - let mut reply = None; - for _ in 0..500 { - client.poll_resends().unwrap(); - if let Some(message) = client.receive_schema_message_once().unwrap() { - reply = Some(message); - break; - } - std::thread::sleep(Duration::from_millis(2)); - } - let reply = reply.unwrap_or_else(|| panic!("{message_id} got no reply")); - let (correlation, answered) = decode_response(&reply).unwrap(); - assert_eq!(&correlation, message_id); - let HuginnMindResponse::Refused(MindRefusal::ResponseTooLarge { bytes, limit }) = - answered.expect("a refusal is an answer, not an envelope failure") - else { - panic!("{message_id} was not refused for its size"); - }; - assert_eq!(limit, MAX_RESPONSE_BYTES); - assert!(bytes > limit, "{message_id} answered {bytes} bytes against a {limit} limit"); + // still unacknowledged, so the fetch is one chunk in flight. + for ((message_id, read), expected) in reads.into_iter().zip(expected) { + let reply = exchange(&mut client, &encode_request(message_id, &read, None).unwrap()).unwrap(); + assert!(encoded_len(&reply) <= MAX_RESPONSE_BYTES, "the deferred answer fits one send"); + assert_eq!(decode_response(&reply).unwrap().0, message_id); + let manifest = deferred_manifest(&reply); + let bytes_expected = rmp_serde::to_vec_named(&expected).unwrap(); + assert_eq!(manifest.size_bytes as usize, bytes_expected.len(), "{message_id}: the manifest is of the answer"); + + let bytes = fetch_content(&manifest, MAX_DEFERRED_BODY_BYTES, |request| exchange(&mut client, &request)) + .unwrap(); + assert_eq!(bytes, bytes_expected, "{message_id}: byte-identical to the answer the mind gave"); + assert_eq!(rmp_serde::from_slice::(&bytes).unwrap(), expected); + assert!(!matches!(expected, HuginnMindResponse::Deferred(_))); } // A real-sized answer that does fit is delivered whole, over the same - // session that was just refused twice: the gate measures the answer, and - // the window it measures against is one the hub actually carries. + // session that just fetched two bodies. let delivered = [ ("m-f", HuginnMindRequest::View { instance: slug(INSTANCE), id: fitting.clone() }), ("m-w", HuginnMindRequest::Whoami), ]; for (message_id, request) in delivered { - client.send_schema_message(&encode_request(message_id, &request, None).unwrap()).unwrap(); - let mut reply = None; - for _ in 0..2000 { - client.poll_resends().unwrap(); - if let Some(message) = client.receive_schema_message_once().unwrap() { - reply = Some(message); - break; - } - std::thread::sleep(Duration::from_millis(2)); - } - let reply = reply.unwrap_or_else(|| panic!("{message_id} got no reply")); + let reply = exchange(&mut client, &encode_request(message_id, &request, None).unwrap()).unwrap(); let (correlation, answered) = decode_response(&reply).unwrap(); assert_eq!(&correlation, message_id); match answered.unwrap() { @@ -666,69 +780,6 @@ mod tests { serving.join().unwrap().0.unwrap(); } - fn encoded_len(message: &CultNetMessage) -> u64 { - encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap().len() as u64 - } - - /// A reply carrying `payload_len` bytes of payload under a chosen - /// `message_id`: the envelope a view answer travels in, with the document - /// replaced by filler, so a size can be chosen instead of found. The - /// `message_id` is part of the encoded envelope too, so pinning the gate - /// against one length alone (every other fixture uses `m-0`) hides a gate - /// that hard-codes that length's contribution instead of measuring it. - fn synthetic_with_id(message_id: &str, payload_len: usize) -> CultNetMessage { - synthetic_full(message_id, "view", "huginn-yggdrasil", payload_len) - } - - /// N4: `operation` and `source_runtime_id` are in the encoded envelope - /// too, exactly as `message_id` is, so a boundary fixture that only ever - /// varies `message_id` cannot tell a gate that genuinely measures the - /// encoded bytes from one that hard-codes a formula shaped to fit that - /// one dimension. `synthetic_with_id` is this with the operation and - /// runtime id `the_gates_boundary_*` tests were pinned to before N4. - fn synthetic_full(message_id: &str, operation: &str, runtime_id: &str, payload_len: usize) -> CultNetMessage { - CultNetMessage::OperationResponse { - message_id: message_id.into(), - service_id: MIND_SERVICE_ID.into(), - operation: operation.into(), - status: "accepted".into(), - payload_schema: MIND_RESPONSE_SCHEMA.into(), - payload_encoding: "messagepack-base64".into(), - payload: "A".repeat(payload_len), - diagnostics: vec![], - source_runtime_id: Some(runtime_id.into()), - } - } - - fn synthetic(payload_len: usize) -> CultNetMessage { - synthetic_with_id("m-0", payload_len) - } - - /// One whose encoded envelope is exactly `target` bytes, under a chosen - /// `message_id`. The envelope's own overhead is measured rather than - /// assumed: the length prefix is the same width on either side of a - /// megabyte, so one probe fixes it. - fn sized_exactly_with_id(message_id: &str, target: u64) -> CultNetMessage { - sized_exactly_full(message_id, "view", "huginn-yggdrasil", target) - } - - /// N4's own probe: `sized_exactly_with_id` generalised over the - /// operation and the runtime id as well as the message id, so a fixture - /// can pin the gate's boundary at a combination a formula shaped to fit - /// `view`/`huginn-yggdrasil` alone was never measured against. - fn sized_exactly_full(message_id: &str, operation: &str, runtime_id: &str, target: u64) -> CultNetMessage { - let probe = 1_200_000_usize; - let at_probe = encoded_len(&synthetic_full(message_id, operation, runtime_id, probe)); - let message = - synthetic_full(message_id, operation, runtime_id, (probe as i64 + (target as i64 - at_probe as i64)) as usize); - assert_eq!(encoded_len(&message), target); - message - } - - fn sized_exactly(target: u64) -> CultNetMessage { - sized_exactly_with_id("m-0", target) - } - /// A connected session whose accept has been acknowledged, so its window /// holds `MAX_PENDING_RELIABLE_PACKETS` and not one less. See the module /// documentation: a test that measures the window exactly and does not @@ -769,150 +820,224 @@ mod tests { (client, session.unwrap()) } - /// The gate's boundary is the transport's, byte for byte. The test above - /// proves the two agree at the sizes real documents reach, which leaves - /// the width of a whole document between the largest answer it delivers - /// and the smallest it refuses; this one leaves nothing. A reply whose - /// encoded envelope is exactly `MAX_RESPONSE_BYTES` passes the gate - /// unchanged and is accepted by a hub `bind` configured; one byte more is - /// refused by both. - /// - /// So the gate measures the encoded envelope, which is what the hub - /// fragments, and not the payload string inside it, which is 231 bytes - /// smaller; and it admits the limit rather than stopping one short of it. - /// Neither is visible to a test whose answers sit tens of thousands of - /// bytes from the boundary. - /// - /// The refusal is measured on the same scale: it is one packet, so the - /// answer that says an answer did not fit always fits itself. And the - /// accepted reply is shown to have filled the window it was measured - /// against, because a one-byte reply after it is refused. - #[test] - fn the_gates_boundary_is_the_transports_boundary_byte_for_byte() { - let at = sized_exactly(MAX_RESPONSE_BYTES); - let over = sized_exactly(MAX_RESPONSE_BYTES + 1); - assert_eq!(within_window(at.clone(), "m-0", "view", "huginn-yggdrasil"), at, "exactly the limit passes"); + /// A reply carrying a chosen payload string, with every field the envelope + /// echoes chosen too, so a size can be chosen instead of found. + fn reply_with_payload(message_id: &str, operation: &str, runtime_id: &str, payload: String) -> CultNetMessage { + CultNetMessage::OperationResponse { + message_id: message_id.into(), + service_id: MIND_SERVICE_ID.into(), + operation: operation.into(), + status: "accepted".into(), + payload_schema: MIND_RESPONSE_SCHEMA.into(), + payload_encoding: "messagepack-base64".into(), + payload, + diagnostics: vec![], + source_runtime_id: Some(runtime_id.into()), + } + } - let refused = within_window(over.clone(), "m-0", "view", "huginn-yggdrasil"); + /// One whose encoded envelope is exactly `target` bytes and whose payload is + /// valid base64, under a `message_id` that is `stem` followed by as many + /// zeros as make the payload a whole number of base64 quanta. The envelope's + /// own overhead is measured, not assumed: the length prefix is the same + /// width on either side of a megabyte, so one probe fixes it. The id is + /// echoed, so it is part of the result. + fn sized_exactly(stem: &str, operation: &str, runtime_id: &str, target: u64) -> (String, CultNetMessage) { + let probe = 1_200_000_usize; + for zeros in 0..4 { + let message_id = format!("{stem}{}", "0".repeat(zeros)); + let at_probe = encoded_len(&reply_with_payload(&message_id, operation, runtime_id, "A".repeat(probe))); + let payload_len = (probe as i64 + (target as i64 - at_probe as i64)) as usize; + if payload_len % 4 == 0 { + let message = reply_with_payload(&message_id, operation, runtime_id, "A".repeat(payload_len)); + assert_eq!(encoded_len(&message), target); + return (message_id, message); + } + } + unreachable!("one of four consecutive id lengths aligns the payload") + } + + /// The boundary is the transport's, byte for byte, at every shape of + /// envelope. A reply whose encoded envelope is exactly `MAX_RESPONSE_BYTES` + /// is delivered unchanged and a hub `bind` configured accepts it; one byte + /// more is deferred, not refused, and a hub refuses it. + /// + /// So the threshold measures the encoded envelope, which is what the hub + /// fragments, and not the payload string inside it, which is a couple of + /// hundred bytes smaller; it admits the limit rather than stopping one + /// short of it; and it is the window itself and not a margin under it. The + /// echoed ids, operations and runtime ids vary independently and in length, + /// so a formula fitted to one envelope shape is measured where it was never + /// fitted. The accepted reply is shown to have filled the window it was + /// measured against, because a one-byte reply after it is refused. + #[test] + fn the_delivery_boundary_is_the_transports_boundary_byte_for_byte() { + let long = format!("m-{}", "1".repeat(38)); + for (stem, operation, runtime_id) in [ + ("m-", "view", "huginn-yggdrasil"), + (long.as_str(), "view", "huginn-yggdrasil"), + ("m-", "whoami", "huginn-thought-cage"), + ("m-", "whoami", "huginn-yggdrasil"), + ("m-", "view", "huginn-thought-cage"), + ] { + let (id_at, at) = sized_exactly(stem, operation, runtime_id, MAX_RESPONSE_BYTES); + let (id_over, over) = sized_exactly(stem, operation, runtime_id, MAX_RESPONSE_BYTES + 1); + let mut bodies = fresh_bodies(); + let case = format!("{operation}/{runtime_id}/{}", stem.len()); + + let delivered = deliver(at.clone(), &id_at, operation, runtime_id, &mut bodies, now()); + assert_eq!(delivered, at, "{case}: exactly the limit is delivered whole"); + assert_eq!(bodies.body_count(), 0, "{case}: nothing was retained for it"); + + let deferred = deliver(over.clone(), &id_over, operation, runtime_id, &mut bodies, now()); + let (correlation, answered) = decode_response(&deferred).unwrap(); + assert_eq!(correlation, id_over, "{case}"); + let Some(HuginnMindResponse::Deferred(DeferredAnswer { manifest })) = answered.ok() else { + panic!("{case}: one byte over the limit was not deferred: {deferred:?}"); + }; + assert_eq!(bodies.body_count(), 1, "{case}"); + assert!( + encoded_len(&deferred) <= MAX_FRAGMENT_BYTES as u64 * 4, + "{case}: the deferred answer is a few packets" + ); + let CultNetMessage::OperationResponse { payload, .. } = &over else { unreachable!() }; + assert_eq!(manifest.size_bytes as usize, STANDARD.decode(payload).unwrap().len(), "{case}"); + + if stem == "m-" && operation == "view" && runtime_id == "huginn-yggdrasil" { + let mut hub = bind("127.0.0.1:0".parse().unwrap(), "huginn-yggdrasil").unwrap(); + let (_client, session) = settled_session(&mut hub); + let error = hub.send_schema_message(&session, &over).unwrap_err(); + assert!(format!("{error:#}").contains("queue is full"), "limit + 1 was accepted: {error:#}"); + hub.send_schema_message(&session, &at).expect("exactly the limit is accepted on an empty window"); + hub.send_schema_message(&session, &reply_with_payload("m-0", "view", runtime_id, "A".into())) + .expect_err("the accepted answer filled the window, so nothing follows it"); + } + } + } + + /// The backstop stays, measured against the deferral bound and on the + /// payload: a body of exactly `MAX_DEFERRED_BODY_BYTES` is deferred, one + /// byte more is refused by name with its own size and that bound, and the + /// deferred answer at the bound still fits one send, so it is never itself + /// deferred. + #[test] + fn an_answer_over_the_deferral_bound_is_refused_by_name() { + let payload_of = |len: u64| STANDARD.encode(vec![0_u8; len as usize]); + let mut bodies = fresh_bodies(); + + let at = reply_with_payload("m-b", "view", "huginn-yggdrasil", payload_of(MAX_DEFERRED_BODY_BYTES)); + let deferred = deliver(at, "m-b", "view", "huginn-yggdrasil", &mut bodies, now()); + let manifest = deferred_manifest(&deferred); + assert_eq!(manifest.size_bytes as u64, MAX_DEFERRED_BODY_BYTES); + assert_eq!(manifest.chunks.len() as u64, MAX_DEFERRED_BODY_BYTES / DEFERRED_CHUNK_BYTES as u64); + assert!(encoded_len(&deferred) <= MAX_RESPONSE_BYTES, "at the bound the deferred answer still fits one send"); + assert_eq!(bodies.body_count(), 1); + drop(bodies); + + let over = reply_with_payload("m-b", "view", "huginn-yggdrasil", payload_of(MAX_DEFERRED_BODY_BYTES + 1)); + let mut empty = fresh_bodies(); + let refused = deliver(over, "m-b", "view", "huginn-yggdrasil", &mut empty, now()); let (correlation, answered) = decode_response(&refused).unwrap(); - assert_eq!(correlation, "m-0"); + assert_eq!(correlation, "m-b"); let HuginnMindResponse::Refused(MindRefusal::ResponseTooLarge { bytes, limit }) = answered.expect("a refusal is an answer, not an envelope failure") else { - panic!("one byte over the limit was not refused: {refused:?}"); + panic!("one byte over the deferral bound was not refused: {refused:?}"); }; - assert_eq!((bytes, limit), (MAX_RESPONSE_BYTES + 1, MAX_RESPONSE_BYTES)); - let refusal_bytes = encoded_len(&refused); + assert_eq!((bytes, limit), (MAX_DEFERRED_BODY_BYTES + 1, MAX_DEFERRED_BODY_BYTES)); + assert!(encoded_len(&refused) <= MAX_FRAGMENT_BYTES as u64, "the refusal is one packet"); + assert_eq!(empty.body_count(), 0, "a refused answer retains nothing"); + } + + /// An oversize reply whose payload cannot be read as the response's bytes + /// is a failure of the envelope, named, and never deferred. + #[test] + fn an_oversize_reply_that_is_not_a_response_is_not_deferred() { + let mut bodies = fresh_bodies(); + let not_base64 = reply_with_payload("m-x", "view", "huginn-yggdrasil", "A".repeat(MAX_RESPONSE_BYTES as usize) + "!"); + let not_a_response = + CultNetMessage::Error { error: "e".repeat(MAX_RESPONSE_BYTES as usize + 1), code: None, details: None }; + for reply in [not_base64, not_a_response] { + assert!(encoded_len(&reply) > MAX_RESPONSE_BYTES); + let answered = deliver(reply, "m-x", "view", "huginn-yggdrasil", &mut bodies, now()); + assert_eq!(rejected(&answered).1, "response-not-encodable"); + } + assert_eq!(bodies.body_count(), 0); + } + + /// Lifecycle end to end through the daemon's own `answer`, on an injected + /// clock: a chunk served inside the TTL slides the expiry, a body untouched + /// for the TTL is gone and its chunk is answered `found: false` by name, + /// and the client asks again and fetches the same answer. + #[test] + fn an_expired_body_answers_found_false_and_the_client_can_ask_again() { + let (_root, mut daemon) = seeded_wide(); + let registry = schema_registry().unwrap(); + let wide = spec_ref(&mut daemon, WIDE_CUT); + let read = HuginnMindRequest::View { instance: slug(INSTANCE), id: wide }; + let expected = rmp_serde::to_vec_named(&daemon.handle(read.clone(), now())).unwrap(); + let t0 = now(); + let after = |seconds: i64| t0 + chrono::Duration::seconds(seconds); + let mut bodies = fresh_bodies(); + + let manifest = ask_deferred(&mut daemon, &mut bodies, "m-v", &read, t0); + assert!(manifest.chunks.len() >= 2, "the fixture needs a second chunk to serve"); + let mut chunk = |index: usize, at: i64| { + answer(&mut daemon, ®istry, &mut bodies, chunk_request(&manifest, index), after(at)) + }; + assert!(found(&chunk(0, 59)), "inside the TTL"); + assert!(found(&chunk(1, 118)), "serving a chunk slid the expiry past the first TTL"); + let gone = chunk(0, 178); + assert!(!found(&gone), "untouched for the TTL since t=118"); + let CultNetMessage::ContentChunkResponse { error, payload, .. } = &gone else { unreachable!() }; + assert!(error.starts_with("FileNotFoundException"), "{error}"); + assert!(payload.is_empty()); + + let again = ask_deferred(&mut daemon, &mut bodies, "m-v2", &read, after(179)); + assert_eq!(again.content_hash, manifest.content_hash, "the same answer is the same body"); + let bytes = fetch_content(&again, MAX_DEFERRED_BODY_BYTES, |request| { + Ok(answer(&mut daemon, ®istry, &mut bodies, request, after(180))) + }) + .unwrap(); + assert_eq!(bytes, expected); + } + + /// Past the budget the body touched longest ago is evicted, its chunks + /// answer `found: false`, and asking again evicts the other in turn. Two + /// identical answers are one body. + #[test] + fn eviction_and_sharing_through_the_daemons_answer() { + let (_root, mut daemon) = seeded_wide(); + let wide = spec_ref(&mut daemon, WIDE_CUT); + let view = HuginnMindRequest::View { instance: slug(INSTANCE), id: wide }; + let query = document_query(); + let size = |daemon: &mut BodyDaemon, read: &HuginnMindRequest| { + rmp_serde::to_vec_named(&daemon.handle(read.clone(), now())).unwrap().len() as u64 + }; + let (view_bytes, query_bytes) = (size(&mut daemon, &view), size(&mut daemon, &query)); + // Room for the larger body alone, and not for both. + let budget = view_bytes.max(query_bytes) + 1; + let t0 = now(); + let after = |seconds: i64| t0 + chrono::Duration::seconds(seconds); + let mut bodies = DeferredBodies::new(budget, TTL); + + let first = ask_deferred(&mut daemon, &mut bodies, "m-1", &view, after(0)); + let same = ask_deferred(&mut daemon, &mut bodies, "m-2", &view, after(1)); + assert_eq!(first.content_hash, same.content_hash, "two identical answers are one body"); + assert_eq!((bodies.body_count(), bodies.stored_bytes()), (1, view_bytes)); + + let second = ask_deferred(&mut daemon, &mut bodies, "m-3", &query, after(2)); + assert_eq!(bodies.body_count(), 1, "the least recently touched body was evicted"); + assert!(bodies.stored_bytes() <= budget); + assert!(!first_chunk_found(&mut daemon, &mut bodies, &first, after(3)), "the evicted body answers found: false"); + assert!(first_chunk_found(&mut daemon, &mut bodies, &second, after(3)), "the newer body stays"); + + let again = ask_deferred(&mut daemon, &mut bodies, "m-4", &view, after(4)); + assert!(first_chunk_found(&mut daemon, &mut bodies, &again, after(5))); assert!( - refusal_bytes <= MAX_FRAGMENT_BYTES as u64, - "the refusal is {refusal_bytes} bytes and must fit one packet" + !first_chunk_found(&mut daemon, &mut bodies, &second, after(5)), + "the client's re-ask evicted the one touched longest ago" ); - - let mut hub = bind("127.0.0.1:0".parse().unwrap(), "huginn-yggdrasil").unwrap(); - let (_client, session) = settled_session(&mut hub); - let error = hub.send_schema_message(&session, &over).unwrap_err(); - assert!(format!("{error:#}").contains("queue is full"), "limit + 1 was accepted: {error:#}"); - hub.send_schema_message(&session, &at).expect("exactly the limit is accepted on an empty window"); - hub.send_schema_message(&session, &synthetic(1)) - .expect_err("the accepted answer filled the window, so nothing follows it"); - } - - /// The boundary above is pinned for one envelope shape: every reply in it - /// carries `message_id` `m-0`, three bytes, so a gate that measures - /// `payload.len() + 231` — exactly the encoded envelope's overhead for - /// that one id — passes it unnoticed. The reply echoes the client's own - /// `message_id`, so that overhead is under the client's control: at a - /// longer id, such a gate is wrong, and wrong on the side that lets an - /// oversize answer through the size check to a send that then fails - /// silently. Forty bytes is used because it is far from `m-0`'s length in - /// either direction, not chosen to sit near some other boundary. - #[test] - fn the_gates_boundary_holds_for_a_second_message_id_length() { - let long_id = format!("m-{}", "0".repeat(38)); - assert_eq!(long_id.len(), 40, "a message_id far from m-0's own length"); - - let at = sized_exactly_with_id(&long_id, MAX_RESPONSE_BYTES); - let over = sized_exactly_with_id(&long_id, MAX_RESPONSE_BYTES + 1); - assert_eq!( - within_window(at.clone(), &long_id, "view", "huginn-yggdrasil"), - at, - "exactly the limit passes at this id's length too" - ); - - let refused = within_window(over.clone(), &long_id, "view", "huginn-yggdrasil"); - let (correlation, answered) = decode_response(&refused).unwrap(); - assert_eq!(correlation, long_id); - let HuginnMindResponse::Refused(MindRefusal::ResponseTooLarge { bytes, limit }) = - answered.expect("a refusal is an answer, not an envelope failure") - else { - panic!("one byte over the limit was not refused: {refused:?}"); - }; - assert_eq!((bytes, limit), (MAX_RESPONSE_BYTES + 1, MAX_RESPONSE_BYTES)); - } - - /// N4: the boundary above is pinned at one `message_id` length, but - /// `operation` and `source_runtime_id` are in the encoded envelope the - /// same way `message_id` is, and every fixture before this one held both - /// fixed at `"view"` and `"huginn-yggdrasil"`. A gate whose measurement is - /// a formula of `message_id`'s length alone -- shaped to pass at exactly - /// `m-0`'s three bytes and the other fixture's forty -- has no way to be - /// wrong there and every way to be wrong once the operation or the - /// runtime id it never looked at changes length too. This holds the - /// boundary at `whoami` (six bytes, not `view`'s four) over a second - /// runtime id, so such a formula is measured somewhere it was never - /// fitted. - #[test] - fn the_gates_boundary_holds_across_operation_and_runtime_id() { - for (operation, runtime_id) in [("view", "huginn-yggdrasil"), ("whoami", "huginn-thought-cage")] { - let at = sized_exactly_full("m-g", operation, runtime_id, MAX_RESPONSE_BYTES); - let over = sized_exactly_full("m-g", operation, runtime_id, MAX_RESPONSE_BYTES + 1); - assert_eq!( - within_window(at.clone(), "m-g", operation, runtime_id), - at, - "{operation}/{runtime_id}: exactly the limit passes" - ); - - let refused = within_window(over.clone(), "m-g", operation, runtime_id); - let (correlation, answered) = decode_response(&refused).unwrap(); - assert_eq!(correlation, "m-g"); - let HuginnMindResponse::Refused(MindRefusal::ResponseTooLarge { bytes, limit }) = - answered.expect("a refusal is an answer, not an envelope failure") - else { - panic!("{operation}/{runtime_id}: one byte over the limit was not refused: {refused:?}"); - }; - assert_eq!((bytes, limit), (MAX_RESPONSE_BYTES + 1, MAX_RESPONSE_BYTES), "{operation}/{runtime_id}"); - } - } - - /// Every fixture up to this one, including the pair above, - /// moves `operation` and the runtime id together -- both short (`view`, - /// 4 bytes / `huginn-yggdrasil`, 16) or both long (`whoami`, 6 / - /// `huginn-thought-cage`, 19) -- so a formula that tracks only one of the - /// two and ignores the other still lands on the right answer at both - /// points by coincidence. This pins a point where they move apart: a long - /// operation with the short runtime id, and a short operation with the - /// long one, so a gate that dropped either term is wrong at one of them. - #[test] - fn the_gates_boundary_holds_when_operation_and_runtime_id_vary_independently() { - for (operation, runtime_id) in [("whoami", "huginn-yggdrasil"), ("view", "huginn-thought-cage")] { - let at = sized_exactly_full("m-g", operation, runtime_id, MAX_RESPONSE_BYTES); - let over = sized_exactly_full("m-g", operation, runtime_id, MAX_RESPONSE_BYTES + 1); - assert_eq!( - within_window(at.clone(), "m-g", operation, runtime_id), - at, - "{operation}/{runtime_id}: exactly the limit passes" - ); - - let refused = within_window(over.clone(), "m-g", operation, runtime_id); - let (correlation, answered) = decode_response(&refused).unwrap(); - assert_eq!(correlation, "m-g"); - let HuginnMindResponse::Refused(MindRefusal::ResponseTooLarge { bytes, limit }) = - answered.expect("a refusal is an answer, not an envelope failure") - else { - panic!("{operation}/{runtime_id}: one byte over the limit was not refused: {refused:?}"); - }; - assert_eq!((bytes, limit), (MAX_RESPONSE_BYTES + 1, MAX_RESPONSE_BYTES), "{operation}/{runtime_id}"); - } } /// One cut spec's own reference, by its cut label. diff --git a/crates/huginn-mind/src/lib.rs b/crates/huginn-mind/src/lib.rs index 14c5d1f..451e73f 100644 --- a/crates/huginn-mind/src/lib.rs +++ b/crates/huginn-mind/src/lib.rs @@ -48,7 +48,7 @@ pub use receipt::{DocumentVersion, Faculty, HuginnCommitReceipt, PipelineProvena pub use refusal::MindRefusal; pub use store::{MindStore, OwnedRedbMessagePackBackingStore}; pub use wire::{ - HuginnMindRequest, HuginnMindResponse, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON, MIND_RESPONSE_SCHEMA, + DeferredAnswer, HuginnMindRequest, HuginnMindResponse, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON, MIND_RESPONSE_SCHEMA, MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, MindStatus, }; diff --git a/crates/huginn-mind/src/refusal.rs b/crates/huginn-mind/src/refusal.rs index f0174f2..ae369d0 100644 --- a/crates/huginn-mind/src/refusal.rs +++ b/crates/huginn-mind/src/refusal.rs @@ -55,14 +55,15 @@ pub enum MindRefusal { FindingWithoutEvidence, UnknownInvariant { label: String }, NotStewarded { repo: String }, - /// The one refusal a mind never raises: the answer does not fit the - /// transport that would have carried it. It lives here because a refusal - /// rides the response schema and there is one of those, so a client reads - /// it the way it reads every other refusal rather than parsing a second - /// vocabulary. `bytes` is the encoded answer and `limit` is what one send - /// may carry, both so a caller can narrow its own request; nothing is - /// truncated or paginated on its behalf. The daemon owns it and the - /// transport's own bound decides it; no rule of the mind is involved. + /// The one refusal a mind never raises: the answer exceeds the largest + /// body the organ will deliver by any plane, one send or a deferred body. + /// It lives here because a refusal rides the response schema and there is + /// one of those, so a client reads it the way it reads every other refusal + /// rather than parsing a second vocabulary. `bytes` is the encoded answer + /// (the payload, not its envelope) and `limit` is the largest body the + /// daemon defers, both so a caller can narrow its own request; nothing is + /// truncated or paginated on its behalf. The daemon owns it and its + /// deferral bound decides it; no rule of the mind is involved. ResponseTooLarge { bytes: u64, limit: u64 }, /// A selection the substrate or the organ's value door refused: `field` /// is the selection field the client sent, `value` the offending part. diff --git a/crates/huginn-mind/src/wire.rs b/crates/huginn-mind/src/wire.rs index d39749e..5583016 100644 --- a/crates/huginn-mind/src/wire.rs +++ b/crates/huginn-mind/src/wire.rs @@ -5,7 +5,7 @@ //! names a transport. use cultcache_rs::DatabaseEntry; -use cultnet_rs::Selection; +use cultnet_rs::{CultMeshCdnArtifactManifest, Selection}; use epiphany_pipeline::{PIPELINE_SCHEMA_EPOCH, PipelineKind, PipelineRef, Slug}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -41,6 +41,18 @@ fn selection_ref(_: &mut schemars::SchemaGenerator) -> schemars::Schema { }) } +/// The deferred answer's manifest is CultMesh's own type, a positional array +/// the C# reference defines (`CultMeshCdnArtifactManifest`, schema version +/// `gamecult.mesh.cdn_artifact_manifest.v1`). CultLib publishes no JSON schema +/// for it, so the schema says what the reference says: an array of its ten +/// fields in order, and nothing here copies their meaning. +fn manifest_array(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "description": "gamecult.mesh.cdn_artifact_manifest.v1, the C# reference's CultMeshCdnArtifactManifest as its positional array: [artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc, chunks[[chunkHash, offset, sizeBytes, recordKey]], tags, metadata].", + "type": "array" + }) +} + /// Every operation a mind answers. `Admit` carries the batch whole because the /// batch already names the instance and the asker; a second `instance` beside /// it would be two declarations. Every read names the instance because ruling @@ -91,6 +103,22 @@ pub enum HuginnMindResponse { View(Option), Query(PipelineSelectionPage), Refused(MindRefusal), + /// The answer did not fit one send and is being held for the client to + /// fetch. Nothing in a mind produces this: the daemon's serve module does, + /// in place of an answer, and the answer it stands for is the named + /// MessagePack of the `HuginnMindResponse` the mind gave. + Deferred(DeferredAnswer), +} + +/// Where the body of an answer that did not fit one send can be fetched: the +/// reference's own manifest, whose `contentHash` is the SHA-256 of the answer's +/// encoded bytes and whose chunks are asked for, one at a time, on the session +/// this arrived on. Row-level facts come from asking again with a header +/// projection; there is no summary. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub struct DeferredAnswer { + #[schemars(schema_with = "manifest_array")] + pub manifest: CultMeshCdnArtifactManifest, } impl HuginnMindResponse { @@ -176,6 +204,11 @@ mod tests { edges: None, }), HuginnMindResponse::Refused(refusal), + HuginnMindResponse::Deferred(DeferredAnswer { + manifest: cultnet_rs::pack_content("huginn.mind_response", "package", "", "", "", b"body", 4) + .unwrap() + .0, + }), ] } diff --git a/schemas/cultnet/huginn.mind_response.v1.schema.json b/schemas/cultnet/huginn.mind_response.v1.schema.json index 39cc870..6164683 100644 --- a/schemas/cultnet/huginn.mind_response.v1.schema.json +++ b/schemas/cultnet/huginn.mind_response.v1.schema.json @@ -69,6 +69,19 @@ "required": [ "Refused" ] + }, + { + "description": "The answer did not fit one send and is being held for the client to\nfetch. Nothing in a mind produces this: the daemon's serve module does,\nin place of an answer, and the answer it stands for is the named\nMessagePack of the `HuginnMindResponse` the mind gave.", + "type": "object", + "properties": { + "Deferred": { + "$ref": "#/$defs/DeferredAnswer" + } + }, + "additionalProperties": false, + "required": [ + "Deferred" + ] } ], "$defs": { @@ -287,6 +300,19 @@ "type": "string", "maxLength": 10 }, + "DeferredAnswer": { + "description": "Where the body of an answer that did not fit one send can be fetched: the\nreference's own manifest, whose `contentHash` is the SHA-256 of the answer's\nencoded bytes and whose chunks are asked for, one at a time, on the session\nthis arrived on. Row-level facts come from asking again with a header\nprojection; there is no summary.", + "type": "object", + "properties": { + "manifest": { + "description": "gamecult.mesh.cdn_artifact_manifest.v1, the C# reference's CultMeshCdnArtifactManifest as its positional array: [artifactId, kind, version, contentHash, sizeBytes, mimeType, createdAtUtc, chunks[[chunkHash, offset, sizeBytes, recordKey]], tags, metadata].", + "type": "array" + } + }, + "required": [ + "manifest" + ] + }, "Deviation": { "type": "object", "properties": { @@ -1202,7 +1228,7 @@ ] }, { - "description": "The one refusal a mind never raises: the answer does not fit the\ntransport that would have carried it. It lives here because a refusal\nrides the response schema and there is one of those, so a client reads\nit the way it reads every other refusal rather than parsing a second\nvocabulary. `bytes` is the encoded answer and `limit` is what one send\nmay carry, both so a caller can narrow its own request; nothing is\ntruncated or paginated on its behalf. The daemon owns it and the\ntransport's own bound decides it; no rule of the mind is involved.", + "description": "The one refusal a mind never raises: the answer exceeds the largest\nbody the organ will deliver by any plane, one send or a deferred body.\nIt lives here because a refusal rides the response schema and there is\none of those, so a client reads it the way it reads every other refusal\nrather than parsing a second vocabulary. `bytes` is the encoded answer\n(the payload, not its envelope) and `limit` is the largest body the\ndaemon defers, both so a caller can narrow its own request; nothing is\ntruncated or paginated on its behalf. The daemon owns it and its\ndeferral bound decides it; no rule of the mind is involved.", "type": "object", "properties": { "ResponseTooLarge": { From 15fc24d6c618e9eddd91747d2997592b92569192 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 07:45:45 +0200 Subject: [PATCH 002/520] BP-3 2: pin the deferral constants Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/bodies.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/huginn-daemon/src/bodies.rs b/crates/huginn-daemon/src/bodies.rs index 1ec113e..f58d05c 100644 --- a/crates/huginn-daemon/src/bodies.rs +++ b/crates/huginn-daemon/src/bodies.rs @@ -156,6 +156,16 @@ mod tests { use super::*; use cultnet_rs::pack_content; + /// The three numbers a client author is told, spelled out: a chunk of + /// 256 KiB, a largest deferred body of 64 MiB, a budget of 256 MiB that + /// holds at least one whole body. + #[test] + fn the_deferral_constants_are_the_documented_numbers() { + assert_eq!(DEFERRED_CHUNK_BYTES, 262_144); + assert_eq!(MAX_DEFERRED_BODY_BYTES, 67_108_864); + assert_eq!(DEFERRED_BUDGET_BYTES, 268_435_456); + } + fn at(seconds: i64) -> DateTime { DateTime::from_timestamp(1_800_000_000 + seconds, 0).unwrap() } From 78f8c6ee2f2a667f67b87788d5ade0648bd52713 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 08:05:26 +0200 Subject: [PATCH 003/520] Cut 11a: the index write side huginn-mind::index owns what of a document is searchable: index_text is a pure function per kind, Mind::index_entries builds the entries with the writing receipt's ordinal, INDEX_TEXT_VERSION says when the rule changed. huginn-daemon::index owns the projection. A Projector reconciles the mind against the collection at startup (the only place a point is judged missing: set difference on point ids, rebuild when the model name, digest, dimensions or text version differ, refuse a collection that is not this instance's) and one worker thread embeds and upserts in batches of 32, retrying with a 30 s to 10 min backoff. IndexSink::committed builds entries and leaves them in an inbox; admission never waits on the embedder or on Qdrant. Whoami reports the worker's health as IndexStatus. The Ollama and Qdrant adapters are the only files holding HTTP or JSON (ureq 3, no TLS). The daemon now requires --qdrant-url, --ollama-url and --embedding-model; NoIndex leaves the binary. Co-Authored-By: Claude Sonnet 5 --- Cargo.lock | 64 +- README.md | 12 +- crates/huginn-daemon/Cargo.toml | 8 + crates/huginn-daemon/src/daemon.rs | 104 ++-- crates/huginn-daemon/src/envelope.rs | 1 + crates/huginn-daemon/src/index/fakes.rs | 177 ++++++ crates/huginn-daemon/src/index/mod.rs | 552 ++++++++++++++++++ crates/huginn-daemon/src/index/ollama.rs | 74 +++ crates/huginn-daemon/src/index/qdrant.rs | 132 +++++ crates/huginn-daemon/src/index/tests.rs | 545 +++++++++++++++++ crates/huginn-daemon/src/lib.rs | 10 +- crates/huginn-daemon/src/main.rs | 7 +- crates/huginn-daemon/src/serve.rs | 127 +++- crates/huginn-mind/src/index.rs | 312 ++++++++++ crates/huginn-mind/src/lib.rs | 4 +- crates/huginn-mind/src/query.rs | 20 +- crates/huginn-mind/src/wire.rs | 30 +- .../huginn.mind_response.v1.schema.json | 128 +++- 18 files changed, 2219 insertions(+), 88 deletions(-) create mode 100644 crates/huginn-daemon/src/index/fakes.rs create mode 100644 crates/huginn-daemon/src/index/mod.rs create mode 100644 crates/huginn-daemon/src/index/ollama.rs create mode 100644 crates/huginn-daemon/src/index/qdrant.rs create mode 100644 crates/huginn-daemon/src/index/tests.rs create mode 100644 crates/huginn-mind/src/index.rs diff --git a/Cargo.lock b/Cargo.lock index 341aa15..3ecdc74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.4.6" @@ -445,6 +451,22 @@ dependencies = [ "digest", ] +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "huginn-daemon" version = "0.1.0" @@ -457,8 +479,11 @@ dependencies = [ "rmp-serde", "schemars", "serde", + "serde_json", + "sha2", "signal-hook", "tempfile", + "ureq", ] [[package]] @@ -573,6 +598,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -967,7 +998,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -995,6 +1026,37 @@ dependencies = [ "subtle", ] +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64", + "log", + "percent-encoding", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "uuid" version = "1.26.1" diff --git a/README.md b/README.md index 2317aa2..695c5ff 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,15 @@ Upstream: `https://github.com/GameCult/Huginn.git` mind over CultNet RUDP: admission and the read side ride `cultnet.operation_request.v0` and `cultnet.operation_response.v0`, and the two wire schemas answer a schema catalog request. -- Retrieval is to depend on Qdrant directly, refusing loudly when Qdrant is - unreachable rather than falling back to a second store or a second writer. - No crate opens that connection yet. +- Retrieval depends on Qdrant directly, with no second store and no second + writer. `huginn-daemon` keeps one collection per mind, `huginn_mind_`, + as a projection of the mind: one worker thread embeds each indexable + document through Ollama and writes its point, off the serving path, and a + restart compares the mind with the collection and writes what is missing. An + unreachable Qdrant or Ollama shows in `whoami` as the index's status and is + retried; admission never waits on it. The daemon requires `--qdrant-url`, + `--ollama-url` and `--embedding-model`. The semantic read side is not wired + yet. - Mind state is not version-controlled. It lives in Huginn's store, not in any repository. diff --git a/crates/huginn-daemon/Cargo.toml b/crates/huginn-daemon/Cargo.toml index 04dfd1a..428588e 100644 --- a/crates/huginn-daemon/Cargo.toml +++ b/crates/huginn-daemon/Cargo.toml @@ -26,7 +26,15 @@ huginn-mind = { path = "../huginn-mind" } rmp-serde = "1" schemars = "1" serde = { version = "1", features = ["derive"] } +# The vector store's and the embedder's HTTP APIs are the xenos boundary, and +# JSON exists in `index/ollama.rs` and `index/qdrant.rs` only. `ureq` 3 with +# default features off is plain HTTP with no TLS (`rustls` and `gzip` are its +# only defaults), which is all the private mesh needs. It stays below 3.4, +# whose `base64` 0.23 would put a second `base64` in a tree that has one. +serde_json = "1" +sha2 = "0.10" signal-hook = "0.3" +ureq = { version = "~3.3", default-features = false } [dev-dependencies] tempfile = "3" diff --git a/crates/huginn-daemon/src/daemon.rs b/crates/huginn-daemon/src/daemon.rs index a1abc28..5742e61 100644 --- a/crates/huginn-daemon/src/daemon.rs +++ b/crates/huginn-daemon/src/daemon.rs @@ -2,28 +2,25 @@ //! rule. `now` is the caller's, so a test pins it and the loop is the only //! thing that reads a wall clock. -use std::path::Path; - use chrono::{DateTime, Utc}; use huginn_mind::epiphany_pipeline::{PipelineRef, Slug}; -use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse}; -use huginn_mind::{Mind, MindRefusal, MindStore, OwnedRedbMessagePackBackingStore, PipelineAdmissionOutcome}; +use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, IndexStatus}; +use huginn_mind::{Mind, MindStore, PipelineAdmissionOutcome}; -/// Where Cut 11's index attaches. Called after a batch commits, with the refs -/// the batch landed, derived writes included; the sink may read each through +/// Where the index attaches. Called after a batch commits, with the refs the +/// batch landed, derived writes included; the sink may read each through /// `mind.view`. Its error is logged and never reaches the caller: the index -/// does not decide whether a batch was admitted. +/// does not decide whether a batch was admitted, and the call must not wait on +/// the index's transport. `status` is what `whoami` reports of it. pub trait IndexSink { fn committed(&mut self, mind: &Mind, writes: &[PipelineRef]) -> anyhow::Result<()>; + fn status(&self) -> IndexStatus; } -/// This cut's index: there is none. -pub struct NoIndex; - -impl IndexSink for NoIndex { - fn committed(&mut self, _mind: &Mind, _writes: &[PipelineRef]) -> anyhow::Result<()> { - Ok(()) - } +/// The envelope's `source_runtime_id` on every response a daemon serving this +/// instance sends. Also what the socket is bound under, before a daemon exists. +pub fn runtime_id(instance: &Slug) -> String { + format!("huginn-{}", instance.0) } /// One mind and whatever indexes what lands in it. @@ -32,17 +29,14 @@ pub struct Daemon> { index: I, } -impl Daemon { - /// Opens the instance's mind, taking the store's exclusive lock for the - /// daemon's lifetime. Ruling 15: a mind that will not open is a refusal - /// here, before any socket exists; this function takes no address and - /// cannot bind one. - pub fn open(state_root: &Path, instance: &Slug) -> Result { - Ok(Self { mind: Mind::open(state_root, instance)?, index: NoIndex }) - } -} - impl> Daemon { + /// A daemon over a mind that is already open. `Mind::open` takes the + /// store's exclusive lock for the daemon's lifetime, and ruling 15 has it + /// refuse before any socket exists: the caller opens the mind first. + pub fn new(mind: Mind, index: I) -> Self { + Self { mind, index } + } + pub fn with_index>(self, index: J) -> Daemon { Daemon { mind: self.mind, index } } @@ -51,9 +45,13 @@ impl> Daemon { &self.mind } + pub fn index(&self) -> &I { + &self.index + } + /// The envelope's `source_runtime_id` on every response this daemon sends. pub fn runtime_id(&self) -> String { - format!("huginn-{}", self.mind.instance().0) + runtime_id(self.mind.instance()) } /// One request, one answer. `Admit` goes to the mind whole, because the @@ -70,7 +68,7 @@ impl> Daemon { return HuginnMindResponse::Refused(refusal); } match request { - HuginnMindRequest::Whoami => HuginnMindResponse::Whoami(self.mind.status()), + HuginnMindRequest::Whoami => HuginnMindResponse::Whoami(self.mind.status(self.index.status())), HuginnMindRequest::Admit(batch) => { let outcome = self.mind.admit(batch, now); if let PipelineAdmissionOutcome::Committed { writes, .. } = &outcome @@ -104,6 +102,8 @@ pub(crate) mod tests { StructuralDelta, Title, VerificationTest, }; use huginn_mind::wire::MindStatus; + use huginn_mind::{MindRefusal, OwnedRedbMessagePackBackingStore}; + use std::path::Path; use cultnet_rs::Selection; use huginn_mind::{Faculty, PipelineAdmissionBatch, PipelineProvenance, PipelinePageItems, PipelineStatus, SemanticQuery}; use std::sync::{Arc, Mutex}; @@ -295,7 +295,7 @@ pub(crate) mod tests { /// real-sized answer either way. pub(crate) fn seeded_wide() -> (TempDir, Daemon) { let root = tempfile::tempdir().unwrap(); - let mut daemon = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let mut daemon = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); for documents in [ campaign_seed(), vec![cut_spec(WIDE_CUT, WIDE_CHANGES), cut_spec(FITTING_CUT, FITTING_CHANGES)], @@ -314,7 +314,7 @@ pub(crate) mod tests { /// removes the store. pub(crate) fn seeded() -> (TempDir, Daemon) { let root = tempfile::tempdir().unwrap(); - let mut daemon = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let mut daemon = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); assert!( matches!(outcome, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), @@ -323,6 +323,27 @@ pub(crate) mod tests { (root, daemon) } + /// No index: for the tests whose subject is not the projection. + pub(crate) struct NoIndex; + + impl IndexSink for NoIndex { + fn committed(&mut self, _mind: &Mind, _writes: &[PipelineRef]) -> anyhow::Result<()> { + Ok(()) + } + + fn status(&self) -> IndexStatus { + IndexStatus::Current + } + } + + /// A daemon over the instance's mind under `state_root`, with no index. + pub(crate) fn open_unindexed( + state_root: &Path, + instance: &Slug, + ) -> Result, MindRefusal> { + Ok(Daemon::new(Mind::open(state_root, instance)?, NoIndex)) + } + #[derive(Clone, Default)] struct RecordingIndex { seen: Arc, Vec)>>>, @@ -335,6 +356,10 @@ pub(crate) mod tests { self.seen.lock().unwrap().push((ids, readable)); Ok(()) } + + fn status(&self) -> IndexStatus { + IndexStatus::Current + } } struct FailingIndex; @@ -343,6 +368,10 @@ pub(crate) mod tests { fn committed(&mut self, _mind: &Mind, _writes: &[PipelineRef]) -> anyhow::Result<()> { Err(anyhow::anyhow!("the index is not there")) } + + fn status(&self) -> IndexStatus { + IndexStatus::Current + } } fn status(daemon: &mut Daemon>) -> MindStatus { @@ -357,7 +386,7 @@ pub(crate) mod tests { #[test] fn a_batch_round_trips_typed_through_handle_without_a_socket() { let root = tempfile::tempdir().unwrap(); - let mut daemon = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let mut daemon = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = outcome else { panic!("expected a commit"); @@ -372,6 +401,7 @@ pub(crate) mod tests { schema_epoch: "epiphany.pipeline.epoch.v2".into(), documents: 1, receipts: 1, + index: IndexStatus::Current, } ); assert_eq!(daemon.runtime_id(), "huginn-yggdrasil"); @@ -406,7 +436,7 @@ pub(crate) mod tests { #[test] fn a_read_or_a_write_naming_another_instance_is_refused_by_the_mind_and_writes_nothing() { let root = tempfile::tempdir().unwrap(); - let mut daemon = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let mut daemon = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); let landed = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = landed else { panic!("expected a commit"); @@ -626,7 +656,7 @@ pub(crate) mod tests { let (question, ruling) = question_and_ruling(); let mut daemon = daemon.with_index(FailingIndex); let root = tempfile::tempdir().unwrap(); - let mut fresh = Daemon::open(root.path(), &slug(INSTANCE)).unwrap().with_index(FailingIndex); + let mut fresh = open_unindexed(root.path(), &slug(INSTANCE)).unwrap().with_index(FailingIndex); fresh.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); let outcome = fresh.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()); let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = outcome else { @@ -639,28 +669,28 @@ pub(crate) mod tests { assert_eq!(status(&mut daemon).documents, 4, "identity, question, ruling and the derived resolution"); } - /// Ruling 15: a mind that will not open is refused, and `Daemon::open` - /// takes no address, so no socket can precede it. `startup`'s order is + /// Ruling 15: a mind that will not open is refused, and opening it takes + /// no address, so no socket can precede it. `startup`'s order is /// pinned in `serve::tests`. #[test] fn the_daemon_refuses_loudly_when_it_cannot_open_the_mind_and_binds_nothing() { let root = tempfile::tempdir().unwrap(); - let held = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let held = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); let path = Mind::::store_path_for(root.path(), &slug(INSTANCE)).unwrap(); assert_eq!( - Daemon::open(root.path(), &slug(INSTANCE)).err(), + open_unindexed(root.path(), &slug(INSTANCE)).err(), Some(MindRefusal::MindAlreadyOwned { path: path.display().to_string() }) ); drop(held); - let mut planted = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let mut planted = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); planted.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); drop(planted); let moved = Mind::::store_path_for(root.path(), &slug(OTHER)).unwrap(); std::fs::create_dir_all(moved.parent().unwrap()).unwrap(); std::fs::copy(&path, &moved).unwrap(); assert_eq!( - Daemon::open(root.path(), &slug(OTHER)).err(), + open_unindexed(root.path(), &slug(OTHER)).err(), Some(MindRefusal::ForeignInstance { declared: OTHER.into(), mind: INSTANCE.into() }) ); } diff --git a/crates/huginn-daemon/src/envelope.rs b/crates/huginn-daemon/src/envelope.rs index b3b0275..fecbde2 100644 --- a/crates/huginn-daemon/src/envelope.rs +++ b/crates/huginn-daemon/src/envelope.rs @@ -183,6 +183,7 @@ mod tests { schema_epoch: "epiphany.pipeline.epoch.v2".into(), documents: 0, receipts: 0, + index: huginn_mind::wire::IndexStatus::Current, }), "accepted", ), diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs new file mode 100644 index 0000000..c63041d --- /dev/null +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -0,0 +1,177 @@ +//! In-memory ports for the projector's tests. Each fake is a handle on shared +//! state, so a test keeps observing after it has moved the fake into a +//! projector or a worker thread. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Condvar, Mutex}; + +use anyhow::{Result, bail}; + +use super::{CollectionMeta, Described, Embedder, ModelIdentity, Point, VectorIndex}; + +pub(crate) fn identity(digest: &str) -> ModelIdentity { + ModelIdentity { name: "model".into(), digest: digest.into(), dimensions: 4 } +} + +/// A door a test holds shut: `embed` announces it has arrived and waits. +#[derive(Clone, Default)] +pub(crate) struct Gate(Arc<(Mutex<(u32, bool)>, Condvar)>); + +impl Gate { + pub(crate) fn shut() -> Self { + Self::default() + } + + /// Blocks until `count` calls to `embed` have arrived at the gate. + pub(crate) fn wait_arrived(&self, count: u32) { + let mut state = self.0.0.lock().unwrap(); + while state.0 < count { + state = self.0.1.wait(state).unwrap(); + } + } + + pub(crate) fn open(&self) { + self.0.0.lock().unwrap().1 = true; + self.0.1.notify_all(); + } + + fn pass(&self) { + let mut state = self.0.0.lock().unwrap(); + state.0 += 1; + self.0.1.notify_all(); + while !state.1 { + state = self.0.1.wait(state).unwrap(); + } + } +} + +pub(crate) struct EmbedderState { + pub(crate) identity: ModelIdentity, + pub(crate) down: bool, + /// The texts of every completed call, in order. + pub(crate) embedded: Vec>, +} + +#[derive(Clone)] +pub(crate) struct FakeEmbedder { + pub(crate) state: Arc>, + gate: Option, +} + +impl FakeEmbedder { + pub(crate) fn new(digest: &str) -> Self { + let state = EmbedderState { identity: identity(digest), down: false, embedded: Vec::new() }; + Self { state: Arc::new(Mutex::new(state)), gate: None } + } + + pub(crate) fn behind(mut self, gate: &Gate) -> Self { + self.gate = Some(gate.clone()); + self + } + + pub(crate) fn texts(&self) -> Vec { + self.state.lock().unwrap().embedded.iter().flatten().cloned().collect() + } +} + +impl Embedder for FakeEmbedder { + fn model_identity(&mut self) -> Result { + let state = self.state.lock().unwrap(); + if state.down { + bail!("the embedder is down"); + } + Ok(state.identity.clone()) + } + + fn embed(&mut self, texts: &[String]) -> Result>> { + if let Some(gate) = &self.gate { + gate.pass(); + } + let mut state = self.state.lock().unwrap(); + if state.down { + bail!("the embedder is down"); + } + state.embedded.push(texts.to_vec()); + let dimensions = state.identity.dimensions as usize; + Ok(texts.iter().map(|text| (0..dimensions).map(|at| (text.len() + at) as f32).collect()).collect()) + } +} + +pub(crate) struct Stored { + pub(crate) label: Described, + pub(crate) points: BTreeMap, +} + +#[derive(Default)] +pub(crate) struct IndexState { + pub(crate) collections: BTreeMap, + pub(crate) down: bool, + pub(crate) recreated: Vec<(String, CollectionMeta)>, + /// The point ids of every upsert call, in order. + pub(crate) upserts: Vec>, +} + +#[derive(Clone, Default)] +pub(crate) struct FakeIndex { + pub(crate) state: Arc>, +} + +impl FakeIndex { + pub(crate) fn holding(&self, collection: &str) -> BTreeSet { + let state = self.state.lock().unwrap(); + state.collections.get(collection).map(|stored| stored.points.keys().cloned().collect()).unwrap_or_default() + } + + /// Plants a collection this organ did not make, or made for another mind. + pub(crate) fn plant(&self, collection: &str, label: Described) { + self.state.lock().unwrap().collections.insert(collection.into(), Stored { label, points: BTreeMap::new() }); + } +} + +impl VectorIndex for FakeIndex { + fn describe(&mut self, collection: &str) -> Result { + let state = self.state.lock().unwrap(); + if state.down { + bail!("the vector store is down"); + } + Ok(state.collections.get(collection).map_or(Described::Absent, |stored| stored.label.clone())) + } + + fn recreate(&mut self, collection: &str, meta: &CollectionMeta) -> Result<()> { + let mut state = self.state.lock().unwrap(); + if state.down { + bail!("the vector store is down"); + } + state.recreated.push((collection.into(), meta.clone())); + state + .collections + .insert(collection.into(), Stored { label: Described::Labelled(meta.clone()), points: BTreeMap::new() }); + Ok(()) + } + + fn ids(&mut self, collection: &str) -> Result> { + let state = self.state.lock().unwrap(); + if state.down { + bail!("the vector store is down"); + } + Ok(state.collections.get(collection).map(|stored| stored.points.keys().cloned().collect()).unwrap_or_default()) + } + + fn upsert(&mut self, collection: &str, points: &[Point]) -> Result<()> { + let mut state = self.state.lock().unwrap(); + if state.down { + bail!("the vector store is down"); + } + state.upserts.push(points.iter().map(|point| point.id.clone()).collect()); + let Some(stored) = state.collections.get_mut(collection) else { bail!("no collection {collection}") }; + for point in points { + stored.points.insert(point.id.clone(), point.clone()); + } + Ok(()) + } +} + +/// A working embedder and vector store, for tests whose subject is elsewhere. +pub(crate) fn pair() -> (FakeEmbedder, FakeIndex) { + (FakeEmbedder::new("d1"), FakeIndex::default()) +} diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs new file mode 100644 index 0000000..e75d7c0 --- /dev/null +++ b/crates/huginn-daemon/src/index/mod.rs @@ -0,0 +1,552 @@ +//! The semantic index's write side: a projection of the mind into a vector +//! collection, kept by one worker thread. +//! +//! The mind is the truth and this is derived. `huginn-mind` decides what a +//! document's text is (`IndexEntry`); this module owns when it is embedded and +//! written, and nothing here is stored in the mind: the worker's health lives +//! in memory and a restart recomputes it. +//! +//! One owner per decision. `Projector::reconcile` is the only place a point is +//! judged missing (the mind against the collection, by set difference, at +//! startup), and the worker is the only writer of points. Live admissions +//! reach the worker through `WorkerSink::committed`, which builds the entries +//! (local and cheap), leaves them in an inbox and returns: admission never +//! waits on an embedder or a vector store. Live entries and startup entries +//! share one embed-and-upsert routine, `Projector::flush_batch`. +//! +//! The ports are traits so the projector is tested with in-memory fakes: +//! `Embedder` (`ollama`) and `VectorIndex` (`qdrant`). Those two files are the +//! xenos boundary; nothing else in the organ speaks HTTP or JSON. +//! +//! Stated limits. The inbox is unbounded: it holds at most the documents +//! admitted while the index is unreachable, each of which is also in the mind. +//! The model identity is read at reconciliation only, so a model re-pulled +//! while the daemon runs takes effect at the next start, and vectors of the +//! wrong length are refused rather than written. + +pub mod ollama; +pub mod qdrant; + +#[cfg(test)] +pub(crate) mod fakes; + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, ensure}; +use huginn_mind::epiphany_pipeline::{PipelineRef, Slug}; +use huginn_mind::wire::IndexStatus; +use huginn_mind::{INDEX_TEXT_VERSION, IndexEntry, Mind, MindStore}; +use sha2::{Digest, Sha256}; + +use crate::daemon::IndexSink; + +/// The `managed_by` a collection carries when this organ made it. +pub const MANAGED_BY: &str = "huginn"; + +/// Documents embedded and written per call. +pub const BATCH: usize = 32; + +/// What an embedder is: the model by name and by digest, and the length of the +/// vectors it returns. Compatibility of a collection is decided on this and +/// on nothing else: never on the address the embedder was reached at. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelIdentity { + pub name: String, + pub digest: String, + pub dimensions: u32, +} + +/// What a collection says about how it was made. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CollectionMeta { + pub managed_by: String, + pub instance: String, + pub model: String, + pub model_digest: String, + pub dimensions: u32, + pub index_text_version: u32, +} + +/// What the vector store holds under a collection's name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Described { + Absent, + /// A collection that carries no metadata this organ wrote. + Unlabelled, + Labelled(CollectionMeta), +} + +/// What travels beside a vector. The id of the point is derived from `doc_id`. +#[derive(Clone, Debug, PartialEq)] +pub struct PointPayload { + pub doc_id: String, + pub kind: String, + pub root: String, + pub ordinal: u64, + pub text_sha256: String, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Point { + pub id: String, + pub vector: Vec, + pub payload: PointPayload, +} + +pub trait Embedder { + fn model_identity(&mut self) -> Result; + /// One vector per text, in order. + fn embed(&mut self, texts: &[String]) -> Result>>; +} + +pub trait VectorIndex { + fn describe(&mut self, collection: &str) -> Result; + /// Drops the collection if it exists and creates it empty with this + /// metadata. + fn recreate(&mut self, collection: &str, meta: &CollectionMeta) -> Result<()>; + /// Every point id the collection holds. + fn ids(&mut self, collection: &str) -> Result>; + fn upsert(&mut self, collection: &str, points: &[Point]) -> Result<()>; +} + +/// The mind's collection: one per instance. +pub fn collection_name(instance: &Slug) -> String { + format!("huginn_mind_{}", instance.0) +} + +/// The point id of a document: the first sixteen bytes of the SHA-256 of the +/// document id, shaped as a version 5 UUID, which is the convention voidbot's +/// collections use. The real id travels in the payload. +pub fn point_id(document_id: &str) -> String { + let mut digest: [u8; 32] = Sha256::digest(document_id.as_bytes()).into(); + digest[6] = (digest[6] & 0x0f) | 0x50; + digest[8] = (digest[8] & 0x3f) | 0x80; + let hex: String = digest[..16].iter().map(|byte| format!("{byte:02x}")).collect(); + format!("{}-{}-{}-{}-{}", &hex[0..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..32]) +} + +/// Why a step did not finish. Both are retried; they differ in what the +/// operator is told. +#[derive(Debug)] +pub enum StepError { + /// The collection is another writer's and is left alone. + Refused(String), + /// The embedder or the vector store did not answer, or answered wrongly. + Unavailable(anyhow::Error), +} + +impl From for StepError { + fn from(error: anyhow::Error) -> Self { + Self::Unavailable(error) + } +} + +impl std::fmt::Display for StepError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Refused(reason) => formatter.write_str(reason), + Self::Unavailable(error) => write!(formatter, "{error:#}"), + } + } +} + +/// Whether a step left work to do. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Advance { + Progressed, + Idle, +} + +/// The projection's state: the entries not yet known to be in the collection, +/// and, once reconciled, the length of vector it accepts. +pub struct Projector { + embedder: E, + index: V, + instance: String, + collection: String, + wanted: BTreeMap, + dimensions: Option, +} + +impl Projector { + pub fn new(embedder: E, index: V, instance: &Slug) -> Self { + Self { + embedder, + index, + instance: instance.0.clone(), + collection: collection_name(instance), + wanted: BTreeMap::new(), + dimensions: None, + } + } + + /// Entries the collection may lack. Idempotent: an entry already wanted is + /// the same document. + pub fn want(&mut self, entries: impl IntoIterator) { + for entry in entries { + self.wanted.insert(entry.id.id.0.clone(), entry); + } + } + + pub fn pending(&self) -> u32 { + u32::try_from(self.wanted.len()).unwrap_or(u32::MAX) + } + + /// One step: reconcile first, then one batch at a time until nothing is + /// wanted. + pub fn advance(&mut self) -> Result { + if self.dimensions.is_none() { + self.reconcile()?; + return Ok(Advance::Progressed); + } + if self.wanted.is_empty() { + return Ok(Advance::Idle); + } + self.flush_batch()?; + Ok(Advance::Progressed) + } + + /// The only place a point is judged missing. Reads the model identity, + /// makes the collection agree with it (creating it, or rebuilding it when + /// the model, its digest, its dimensions or the text version differ), + /// refuses a collection that is not this instance's, then drops from + /// `wanted` every entry whose point the collection already holds. + pub fn reconcile(&mut self) -> Result<(), StepError> { + let identity = self.embedder.model_identity().context("reading the embedding model's identity")?; + let wanted = CollectionMeta { + managed_by: MANAGED_BY.into(), + instance: self.instance.clone(), + model: identity.name.clone(), + model_digest: identity.digest.clone(), + dimensions: identity.dimensions, + index_text_version: INDEX_TEXT_VERSION, + }; + match self.index.describe(&self.collection).context("describing the collection")? { + Described::Absent => self.recreate(&wanted)?, + Described::Unlabelled => { + return Err(StepError::Refused(format!( + "collection {} exists and carries no Huginn metadata; it is not this organ's to rewrite", + self.collection + ))); + } + Described::Labelled(found) if found.managed_by != MANAGED_BY || found.instance != self.instance => { + return Err(StepError::Refused(format!( + "collection {} is managed by {:?} for instance {:?}; this organ serves {:?}", + self.collection, found.managed_by, found.instance, self.instance + ))); + } + Described::Labelled(found) if found != wanted => self.recreate(&wanted)?, + Described::Labelled(_) => {} + } + let present = self.index.ids(&self.collection).context("listing the collection's points")?; + self.wanted.retain(|document_id, _| !present.contains(&point_id(document_id))); + self.dimensions = Some(identity.dimensions); + Ok(()) + } + + fn recreate(&mut self, meta: &CollectionMeta) -> Result<()> { + self.index.recreate(&self.collection, meta).context("creating the collection") + } + + /// The one embed-and-upsert routine, for startup and live entries alike: + /// the first `BATCH` wanted entries are embedded, written, and only then + /// no longer wanted. + pub fn flush_batch(&mut self) -> Result<(), StepError> { + self.flush().map_err(StepError::Unavailable) + } + + fn flush(&mut self) -> Result<()> { + let dimensions = self.dimensions.context("flush before reconcile")? as usize; + let batch: Vec = self.wanted.values().take(BATCH).cloned().collect(); + if batch.is_empty() { + return Ok(()); + } + let texts: Vec = batch.iter().map(|entry| entry.text.clone()).collect(); + let vectors = self.embedder.embed(&texts).context("embedding")?; + ensure!(vectors.len() == batch.len(), "the embedder returned {} vectors for {} texts", vectors.len(), batch.len()); + let mut points = Vec::with_capacity(batch.len()); + for (entry, vector) in batch.iter().zip(vectors) { + ensure!( + vector.len() == dimensions, + "the embedder returned a vector of {} for {} where the collection holds {dimensions}", + vector.len(), + entry.id.id.0 + ); + points.push(Point { + id: point_id(&entry.id.id.0), + vector, + payload: PointPayload { + doc_id: entry.id.id.0.clone(), + kind: entry.id.kind.name().into(), + root: entry.root.clone(), + ordinal: entry.ordinal, + text_sha256: entry.text_sha256.clone(), + }, + }); + } + self.index.upsert(&self.collection, &points).context("writing points")?; + for entry in &batch { + self.wanted.remove(&entry.id.id.0); + } + Ok(()) + } + + /// The health after a step that did not fail. + pub fn progress_status(&self) -> IndexStatus { + match (self.dimensions, self.pending()) { + (None, pending) => IndexStatus::Reconciling { pending }, + (Some(_), 0) => IndexStatus::Current, + (Some(_), pending) => IndexStatus::Behind { pending }, + } + } + + /// The health after `attempts` consecutive failures, the last of them + /// `error`. + pub fn failure_status(&self, error: &StepError, attempts: u32) -> IndexStatus { + let pending = self.pending(); + match error { + StepError::Refused(reason) => IndexStatus::Refused { pending, attempts, reason: reason.clone() }, + StepError::Unavailable(error) => IndexStatus::Failing { pending, attempts, error: format!("{error:#}") }, + } + } +} + +/// The retry delay: it starts at `initial`, doubles per consecutive failure and +/// stops at `max`. +#[derive(Clone, Copy, Debug)] +pub struct Backoff { + pub initial: Duration, + pub max: Duration, +} + +impl Default for Backoff { + fn default() -> Self { + Self { initial: Duration::from_secs(30), max: Duration::from_secs(600) } + } +} + +impl Backoff { + pub fn after(&self, delay: Duration) -> Duration { + (delay * 2).min(self.max) + } +} + +struct Shared { + inbox: Vec, + status: IndexStatus, + closed: bool, +} + +type Lock = Arc<(Mutex, Condvar)>; + +fn lock(shared: &Lock) -> MutexGuard<'_, Shared> { + shared.0.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// With no timeout, blocks until the inbox holds something or the sink is +/// closed. With one, sleeps out the backoff whatever arrives: a failing index +/// is retried on its schedule, not once per admission. +fn wait_for_work(shared: &Lock, timeout: Option) { + let mut guard = lock(shared); + match timeout { + None => { + while guard.inbox.is_empty() && !guard.closed { + guard = shared.1.wait(guard).unwrap_or_else(|poisoned| poisoned.into_inner()); + } + } + Some(timeout) => { + let deadline = Instant::now() + timeout; + while !guard.closed { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return; + } + guard = shared.1.wait_timeout(guard, left).unwrap_or_else(|poisoned| poisoned.into_inner()).0; + } + } + } +} + +fn publish(shared: &Lock, status: IndexStatus) { + lock(shared).status = status; + shared.1.notify_all(); +} + +/// The status with what the worker has not yet taken from the inbox added to +/// its pending count. +fn including(status: &IndexStatus, queued: u32) -> IndexStatus { + use IndexStatus as S; + match status { + S::Reconciling { pending } => S::Reconciling { pending: pending + queued }, + S::Current if queued == 0 => S::Current, + S::Current => S::Behind { pending: queued }, + S::Behind { pending } => S::Behind { pending: pending + queued }, + S::Failing { pending, attempts, error } => { + S::Failing { pending: pending + queued, attempts: *attempts, error: error.clone() } + } + S::Refused { pending, attempts, reason } => { + S::Refused { pending: pending + queued, attempts: *attempts, reason: reason.clone() } + } + } +} + +fn work(mut projector: Projector, shared: Lock, backoff: Backoff) { + let mut failures = 0_u32; + let mut delay = backoff.initial; + loop { + let inbox = { + let mut guard = lock(&shared); + if guard.closed { + return; + } + std::mem::take(&mut guard.inbox) + }; + projector.want(inbox); + // While a failure is being reported, the report stands until an + // attempt succeeds: publishing progress between retries would flicker. + if failures == 0 { + publish(&shared, projector.progress_status()); + } + match projector.advance() { + Ok(Advance::Progressed) => { + failures = 0; + delay = backoff.initial; + publish(&shared, projector.progress_status()); + } + Ok(Advance::Idle) => { + failures = 0; + delay = backoff.initial; + publish(&shared, projector.progress_status()); + wait_for_work(&shared, None); + } + Err(error) => { + failures = failures.saturating_add(1); + eprintln!( + "huginn: the index could not advance (attempt {failures}, {} pending, retrying in {:?}): {error}", + projector.pending(), + delay + ); + publish(&shared, projector.failure_status(&error, failures)); + wait_for_work(&shared, Some(delay)); + delay = backoff.after(delay); + } + } + } +} + +/// The daemon's index: a handle on the one worker thread that owns the ports. +/// `committed` builds entries and queues them; `status` reads the worker's +/// health. Dropping the sink tells the worker to stop and does not wait for +/// it: a call in flight ends with its own timeout or with the process. +pub struct WorkerSink { + shared: Lock, +} + +impl WorkerSink { + /// Starts the worker. `startup` is every indexable document the mind holds + /// now; the worker's first act is to reconcile the collection against it. + pub fn spawn(embedder: E, index: V, instance: &Slug, startup: Vec, backoff: Backoff) -> Self + where + E: Embedder + Send + 'static, + V: VectorIndex + Send + 'static, + { + let mut projector = Projector::new(embedder, index, instance); + projector.want(startup); + let shared: Lock = Arc::new(( + Mutex::new(Shared { inbox: Vec::new(), status: projector.progress_status(), closed: false }), + Condvar::new(), + )); + let worker = Arc::clone(&shared); + std::thread::spawn(move || work(projector, worker, backoff)); + Self { shared } + } + + /// Blocks until the health satisfies `predicate`, and returns it. For + /// tests and probes that must observe the worker without a clock; it gives + /// up after two minutes rather than hang a run. + pub fn wait_status(&self, predicate: impl Fn(&IndexStatus) -> bool) -> IndexStatus { + let mut guard = lock(&self.shared); + loop { + let seen = including(&guard.status, u32::try_from(guard.inbox.len()).unwrap_or(u32::MAX)); + if predicate(&seen) { + return seen; + } + let (next, result) = self + .shared + .1 + .wait_timeout(guard, Duration::from_secs(120)) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + guard = next; + assert!(!result.timed_out(), "the index never reached the awaited state; last seen {:?}", guard.status); + } + } +} + +impl Drop for WorkerSink { + fn drop(&mut self) { + lock(&self.shared).closed = true; + self.shared.1.notify_all(); + } +} + +impl IndexSink for WorkerSink { + fn committed(&mut self, mind: &Mind, writes: &[PipelineRef]) -> Result<()> { + let entries = mind.index_entries(Some(writes))?; + if entries.is_empty() { + return Ok(()); + } + lock(&self.shared).inbox.extend(entries); + self.shared.1.notify_all(); + Ok(()) + } + + fn status(&self) -> IndexStatus { + let guard = lock(&self.shared); + including(&guard.status, u32::try_from(guard.inbox.len()).unwrap_or(u32::MAX)) + } +} + +/// Plain HTTP to one base address, in text: the adapters own what the text +/// means. There is no TLS in this build; the addresses it serves are the +/// private mesh's. +pub(crate) struct Http { + agent: ureq::Agent, + base: String, +} + +#[derive(Clone, Copy)] +pub(crate) enum Method { + Get, + Put, + Post, + Delete, +} + +impl Http { + pub(crate) fn new(base_url: &str, timeout: Duration) -> Self { + let agent: ureq::Agent = + ureq::Agent::config_builder().timeout_global(Some(timeout)).http_status_as_error(false).build().into(); + Self { agent, base: base_url.trim_end_matches('/').to_owned() } + } + + /// The status and the body, whatever the status is. + pub(crate) fn request(&self, method: Method, path: &str, body: Option<&str>) -> Result<(u16, String)> { + let url = format!("{}{path}", self.base); + let json = "application/json"; + let sent = match method { + Method::Get => self.agent.get(&url).call(), + Method::Delete => self.agent.delete(&url).call(), + Method::Post => self.agent.post(&url).header("content-type", json).send(body.unwrap_or_default()), + Method::Put => self.agent.put(&url).header("content-type", json).send(body.unwrap_or_default()), + }; + let mut response = sent.with_context(|| format!("no answer from {url}"))?; + let status = response.status().as_u16(); + let text = response.body_mut().read_to_string().with_context(|| format!("reading the answer from {url}"))?; + Ok((status, text)) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/huginn-daemon/src/index/ollama.rs b/crates/huginn-daemon/src/index/ollama.rs new file mode 100644 index 0000000..850b276 --- /dev/null +++ b/crates/huginn-daemon/src/index/ollama.rs @@ -0,0 +1,74 @@ +//! The embedder port over Ollama's HTTP API. This file and `qdrant.rs` are the +//! xenos boundary: JSON exists here and nowhere else in the organ. + +use std::time::Duration; + +use anyhow::{Context, Result, bail, ensure}; +use serde_json::{Value, json}; + +use super::{Embedder, Http, Method, ModelIdentity}; + +/// The instruction Qwen3 embedding models take in front of a query (never in +/// front of a document). Cut 11b's semantic read prepends it as +/// `Instruct: \nQuery: `. +pub const QUERY_INSTRUCTION: &str = + "Given a question about a project's pipeline state, retrieve the documents that answer it"; + +/// One model on one Ollama. A cold model takes seconds to load, so the +/// timeout is generous; the worker owns the schedule around it. +pub struct OllamaEmbedder { + http: Http, + model: String, +} + +impl OllamaEmbedder { + pub fn new(base_url: &str, model: &str) -> Self { + Self { http: Http::new(base_url, Duration::from_secs(300)), model: model.to_owned() } + } + + fn ask(&self, method: Method, path: &str, body: Option<&Value>) -> Result { + let text = body.map(Value::to_string); + let (status, answer) = self.http.request(method, path, text.as_deref())?; + ensure!(status == 200, "Ollama answered {status} to {path}: {}", answer.chars().take(300).collect::()); + serde_json::from_str(&answer).with_context(|| format!("Ollama's answer to {path} is not JSON")) + } +} + +impl Embedder for OllamaEmbedder { + /// The configured name, the digest Ollama lists for it, and the model's + /// embedding length. The address is not part of the identity. + fn model_identity(&mut self) -> Result { + let tags = self.ask(Method::Get, "/api/tags", None)?; + let listed = tags["models"].as_array().context("Ollama's tag list has no models")?; + let digest = listed + .iter() + .find(|entry| entry["name"] == self.model.as_str() || entry["model"] == self.model.as_str()) + .and_then(|entry| entry["digest"].as_str()) + .with_context(|| format!("Ollama does not list the model {}", self.model))? + .to_owned(); + let shown = self.ask(Method::Post, "/api/show", Some(&json!({ "model": self.model })))?; + let info = shown["model_info"].as_object().context("Ollama's model description has no model_info")?; + let Some(length) = info.iter().find(|(key, _)| key.ends_with(".embedding_length")).and_then(|(_, value)| value.as_u64()) + else { + bail!("Ollama does not say how long {}'s embeddings are", self.model); + }; + let dimensions = u32::try_from(length).context("an embedding length beyond u32")?; + Ok(ModelIdentity { name: self.model.clone(), digest, dimensions }) + } + + fn embed(&mut self, texts: &[String]) -> Result>> { + let answer = self.ask(Method::Post, "/api/embed", Some(&json!({ "model": self.model, "input": texts })))?; + let embeddings = answer["embeddings"].as_array().context("Ollama's answer has no embeddings")?; + embeddings + .iter() + .map(|vector| { + vector + .as_array() + .context("an embedding is not a list")? + .iter() + .map(|number| number.as_f64().map(|value| value as f32).context("an embedding holds a non-number")) + .collect() + }) + .collect() + } +} diff --git a/crates/huginn-daemon/src/index/qdrant.rs b/crates/huginn-daemon/src/index/qdrant.rs new file mode 100644 index 0000000..c119f7e --- /dev/null +++ b/crates/huginn-daemon/src/index/qdrant.rs @@ -0,0 +1,132 @@ +//! The vector-store port over Qdrant's HTTP API. With `ollama.rs`, the xenos +//! boundary: JSON exists here and nowhere else in the organ. + +use std::collections::BTreeSet; +use std::time::Duration; + +use anyhow::{Context, Result, ensure}; +use serde_json::{Value, json}; + +use super::{CollectionMeta, Described, Http, Method, Point, VectorIndex}; + +/// One Qdrant. Every path names a collection this organ was told to own; it +/// never lists or touches another. +pub struct QdrantIndex { + http: Http, +} + +/// The ids fetched per scroll page. +const SCROLL_PAGE: u64 = 1000; + +impl QdrantIndex { + pub fn new(base_url: &str) -> Self { + Self { http: Http::new(base_url, Duration::from_secs(60)) } + } + + fn ask(&self, method: Method, path: &str, body: Option<&Value>, ok: &[u16]) -> Result<(u16, Value)> { + let text = body.map(Value::to_string); + let (status, answer) = self.http.request(method, path, text.as_deref())?; + ensure!(ok.contains(&status), "Qdrant answered {status} to {path}: {}", answer.chars().take(300).collect::()); + let parsed = if answer.is_empty() { Value::Null } else { serde_json::from_str(&answer).unwrap_or(Value::Null) }; + Ok((status, parsed)) + } +} + +fn metadata_json(meta: &CollectionMeta) -> Value { + json!({ + "managed_by": meta.managed_by, + "instance": meta.instance, + "model": meta.model, + "model_digest": meta.model_digest, + "dimensions": meta.dimensions, + "index_text_version": meta.index_text_version, + }) +} + +/// What a collection's `config.metadata` says. A collection whose metadata +/// does not name a writer and an instance is unlabelled; one that does but is +/// missing the rest reads as an empty model, which will not match and will be +/// rebuilt if the writer and the instance are ours. +fn described(metadata: &Value) -> Described { + let text = |key: &str| metadata[key].as_str().map(str::to_owned); + let (Some(managed_by), Some(instance)) = (text("managed_by"), text("instance")) else { + return Described::Unlabelled; + }; + let number = |key: &str| metadata[key].as_u64().and_then(|value| u32::try_from(value).ok()).unwrap_or(0); + Described::Labelled(CollectionMeta { + managed_by, + instance, + model: text("model").unwrap_or_default(), + model_digest: text("model_digest").unwrap_or_default(), + dimensions: number("dimensions"), + index_text_version: number("index_text_version"), + }) +} + +impl VectorIndex for QdrantIndex { + fn describe(&mut self, collection: &str) -> Result { + let (status, answer) = self.ask(Method::Get, &format!("/collections/{collection}"), None, &[200, 404])?; + if status == 404 { + return Ok(Described::Absent); + } + Ok(described(&answer["result"]["config"]["metadata"])) + } + + fn recreate(&mut self, collection: &str, meta: &CollectionMeta) -> Result<()> { + let path = format!("/collections/{collection}"); + self.ask(Method::Delete, &format!("{path}?wait=true"), None, &[200, 404])?; + let body = json!({ + "vectors": { "size": meta.dimensions, "distance": "Cosine", "on_disk": true }, + "metadata": metadata_json(meta), + }); + self.ask(Method::Put, &format!("{path}?wait=true"), Some(&body), &[200])?; + Ok(()) + } + + fn ids(&mut self, collection: &str) -> Result> { + let mut ids = BTreeSet::new(); + let mut offset = Value::Null; + loop { + let mut body = json!({ "limit": SCROLL_PAGE, "with_payload": false, "with_vector": false }); + if !offset.is_null() { + body["offset"] = offset; + } + let (_, answer) = + self.ask(Method::Post, &format!("/collections/{collection}/points/scroll"), Some(&body), &[200])?; + let points = answer["result"]["points"].as_array().context("Qdrant's scroll has no points")?; + for point in points { + ids.insert(point["id"].as_str().context("a point id is not a string")?.to_owned()); + } + offset = answer["result"]["next_page_offset"].clone(); + if offset.is_null() { + return Ok(ids); + } + } + } + + fn upsert(&mut self, collection: &str, points: &[Point]) -> Result<()> { + let points: Vec = points + .iter() + .map(|point| { + json!({ + "id": point.id, + "vector": point.vector, + "payload": { + "doc_id": point.payload.doc_id, + "kind": point.payload.kind, + "root": point.payload.root, + "ordinal": point.payload.ordinal, + "text_sha256": point.payload.text_sha256, + }, + }) + }) + .collect(); + self.ask( + Method::Put, + &format!("/collections/{collection}/points?wait=true"), + Some(&json!({ "points": points })), + &[200], + )?; + Ok(()) + } +} diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs new file mode 100644 index 0000000..f4c17ca --- /dev/null +++ b/crates/huginn-daemon/src/index/tests.rs @@ -0,0 +1,545 @@ +//! The projection's rules, against in-memory ports; the adapters' request and +//! answer shapes, against a stub that speaks just enough HTTP. No test needs a +//! live Qdrant or Ollama. + +use std::collections::BTreeSet; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, IndexStatus}; +use huginn_mind::{Mind, OwnedRedbMessagePackBackingStore, PipelineAdmissionOutcome}; +use tempfile::TempDir; + +use super::fakes::{FakeEmbedder, FakeIndex, Gate, pair}; +use super::ollama::OllamaEmbedder; +use super::qdrant::QdrantIndex; +use super::*; +use crate::daemon::Daemon; +use crate::daemon::tests::{INSTANCE, OTHER, batch, campaign_seed, now, question_and_ruling, slug}; + +fn collection() -> String { + collection_name(&slug(INSTANCE)) +} + +/// A real mind holding a campaign, a question, the ruling that answers it and +/// the resolution admission derived: four indexable documents, beside the +/// identity and the stewardship, which carry no text. +fn mind_with_documents() -> (TempDir, Mind) { + let root = tempfile::tempdir().unwrap(); + let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (question, ruling) = question_and_ruling(); + for documents in [campaign_seed(), vec![question, ruling]] { + let outcome = mind.admit(batch(INSTANCE, documents), now()); + assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); + } + (root, mind) +} + +/// How many times the collection was created and how many upserts it took. +fn counts(index: &FakeIndex) -> (usize, usize) { + let state = index.state.lock().unwrap(); + (state.recreated.len(), state.upserts.len()) +} + +fn drain(projector: &mut Projector) { + while projector.advance().unwrap() == Advance::Progressed {} +} + +fn point_ids(entries: &[IndexEntry]) -> BTreeSet { + entries.iter().map(|entry| point_id(&entry.id.id.0)).collect() +} + +fn projected(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE)); + projector.want(entries.iter().cloned()); + drain(&mut projector); + projector +} + +/// P4's function, as voidbot's `toQdrantPointId` computes it: the first sixteen +/// bytes of the SHA-256, version and variant bits set. The fixed vectors were +/// computed independently of this crate. +#[test] +fn point_ids_are_a_pure_function_of_the_document_id() { + assert_eq!(point_id("eureka-state:question:Q1"), "1e8a661b-4b7e-5245-a9ad-b15a3fa27031"); + assert_eq!(point_id("eureka-state:campaign:self"), "d4135998-051d-57fb-9360-82964874fbc9"); + assert_eq!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q1")); + assert_ne!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q2")); +} + +/// The projection converges by set difference: after a crash that lost two +/// points, a restart embeds and writes those two, and nothing else, and the +/// collection is not rebuilt. +#[test] +fn reconcile_indexes_exactly_the_indexable_documents_the_collection_lacks() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + assert_eq!(entries.len(), 4); + + let (embedder, index) = pair(); + projected(&embedder, &index, &entries); + assert_eq!( + index.holding(&collection()), + point_ids(&entries), + "one point per indexable document, none for the identity or the stewardship" + ); + + let lost: Vec = entries.iter().take(2).cloned().collect(); + { + let mut state = index.state.lock().unwrap(); + let stored = state.collections.get_mut(&collection()).unwrap(); + for entry in &lost { + stored.points.remove(&point_id(&entry.id.id.0)); + } + state.upserts.clear(); + state.recreated.clear(); + } + + let restarted = FakeEmbedder::new("d1"); + projected(&restarted, &index, &entries); + let state = index.state.lock().unwrap(); + assert!(state.recreated.is_empty(), "a collection that agrees is not rebuilt"); + let written: BTreeSet = state.upserts.concat().into_iter().collect(); + assert_eq!(written, point_ids(&lost), "exactly the missing points were written"); + assert_eq!(state.upserts.concat().len(), 2, "each once"); + assert_eq!(restarted.texts().len(), 2, "and only their text was embedded"); + drop(state); + assert_eq!(index.holding(&collection()), point_ids(&entries)); +} + +/// Compatibility is the model by name and digest, its dimensions and the text +/// version. Any of them differing rebuilds the collection whole, and no vector +/// of one model survives beside another's. +#[test] +fn a_model_digest_change_rebuilds_the_collection() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let changes: [(&str, Box); 3] = [ + ("digest", Box::new(|identity| identity.digest = "d2".into())), + ("name", Box::new(|identity| identity.name = "other-model".into())), + ("dimensions", Box::new(|identity| identity.dimensions = 8)), + ]; + for (what, change) in changes { + let (embedder, index) = pair(); + projected(&embedder, &index, &entries); + assert_eq!(index.state.lock().unwrap().recreated.len(), 1, "the first run creates the collection"); + + let second = FakeEmbedder::new("d1"); + change(&mut second.state.lock().unwrap().identity); + let expected = second.state.lock().unwrap().identity.clone(); + projected(&second, &index, &entries); + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2, "a changed {what} rebuilds"); + let (_, meta) = state.recreated.last().unwrap(); + assert_eq!( + (&meta.model, &meta.model_digest, meta.dimensions), + (&expected.name, &expected.digest, expected.dimensions), + "{what}" + ); + assert_eq!(second.texts().len(), 4, "every document is embedded again under {what}"); + assert_eq!(state.collections[&collection()].points.len(), 4); + } + + // The same model, digest and dimensions: nothing is rebuilt or rewritten. + let (embedder, index) = pair(); + projected(&embedder, &index, &entries); + let same = FakeEmbedder::new("d1"); + projected(&same, &index, &entries); + assert_eq!(index.state.lock().unwrap().recreated.len(), 1); + assert!(same.texts().is_empty()); + + // A text version the collection was not built with is a rebuild too. + { + let mut state = index.state.lock().unwrap(); + let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; + meta.index_text_version += 1; + } + projected(&FakeEmbedder::new("d1"), &index, &entries); + assert_eq!(index.state.lock().unwrap().recreated.len(), 2); +} + +/// No cross-mind writes: a collection that another writer or another instance +/// made, or that carries no label, is left exactly as it was. +#[test] +fn a_collection_owned_by_another_instance_is_never_recreated() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let ours = |managed_by: &str, instance: &str| CollectionMeta { + managed_by: managed_by.into(), + instance: instance.into(), + model: "model".into(), + model_digest: "d1".into(), + dimensions: 4, + index_text_version: INDEX_TEXT_VERSION, + }; + for foreign in [ + Described::Unlabelled, + Described::Labelled(ours(MANAGED_BY, OTHER)), + Described::Labelled(ours("voidbot", INSTANCE)), + ] { + let (embedder, index) = pair(); + index.plant(&collection(), foreign.clone()); + let sentinel = point_id("someone-elses-document"); + index.state.lock().unwrap().collections.get_mut(&collection()).unwrap().points.insert( + sentinel.clone(), + Point { + id: sentinel.clone(), + vector: vec![0.0; 4], + payload: PointPayload { + doc_id: "someone-elses-document".into(), + kind: "x".into(), + root: "x".into(), + ordinal: 1, + text_sha256: "x".into(), + }, + }, + ); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE)); + projector.want(entries.iter().cloned()); + let error = projector.reconcile().expect_err("a foreign collection is refused"); + assert!(matches!(error, StepError::Refused(_)), "{foreign:?}: {error}"); + assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Refused { pending: 4, .. })); + let state = index.state.lock().unwrap(); + assert!(state.recreated.is_empty() && state.upserts.is_empty(), "{foreign:?} was written to"); + assert_eq!(state.collections[&collection()].label, foreign); + assert!(state.collections[&collection()].points.contains_key(&sentinel), "{foreign:?} lost a point"); + assert!(embedder.texts().is_empty()); + } +} + +/// A vector of the wrong length is refused before anything is written, and the +/// documents stay wanted. +#[test] +fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE)); + projector.want(entries.iter().cloned()); + projector.reconcile().unwrap(); + embedder.state.lock().unwrap().identity.dimensions = 8; + let error = projector.flush_batch().expect_err("a longer vector is refused"); + assert!(matches!(error, StepError::Unavailable(_)), "{error}"); + assert_eq!(projector.pending(), 4); + assert!(index.state.lock().unwrap().upserts.is_empty()); +} + +fn whoami_index(daemon: &mut Daemon) -> IndexStatus { + match daemon.handle(HuginnMindRequest::Whoami, now()) { + HuginnMindResponse::Whoami(status) => status.index, + other => panic!("expected a status, got {other:?}"), + } +} + +fn committed(response: HuginnMindResponse) { + assert!( + matches!(response, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), + "{response:?}" + ); +} + +fn quick() -> Backoff { + Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2) } +} + +/// The index never delays or refuses admission: the embedder is shut, the +/// worker is inside it, and two more admissions are answered at once. The gate +/// counts arrivals, so this observes where the work is and not how long it +/// took. +#[test] +fn an_admission_replies_while_the_embedder_is_blocked() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let index = FakeIndex::default(); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.index_entries(None).unwrap(), quick()); + let mut daemon = Daemon::new(mind, sink); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, campaign_seed())), now())); + gate.wait_arrived(1); + let (question, ruling) = question_and_ruling(); + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now())); + assert_eq!( + whoami_index(&mut daemon), + IndexStatus::Behind { pending: 4 }, + "the campaign is in the embedder and three more are queued" + ); + assert!(index.holding(&collection()).is_empty(), "nothing has been written yet"); + + gate.open(); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 4); + assert_eq!(whoami_index(&mut daemon), IndexStatus::Current); +} + +/// A failing embedder is reported by `whoami`, retried on the backoff, and +/// nothing is dropped: when it returns, every document is written. +#[test] +fn a_failing_embedder_is_visible_in_whoami_and_retried_never_dropped() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (embedder, index) = pair(); + let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), Vec::new(), quick()); + let mut daemon = Daemon::new(mind, sink); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + + embedder.state.lock().unwrap().down = true; + let (question, ruling) = question_and_ruling(); + for documents in [campaign_seed(), vec![question, ruling]] { + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), now())); + } + let seen = daemon.index().wait_status(|status| matches!(status, IndexStatus::Failing { attempts, .. } if *attempts >= 2)); + let IndexStatus::Failing { pending, error, .. } = seen else { panic!() }; + assert_eq!(pending, 4); + assert!(error.contains("the embedder is down"), "{error}"); + assert!(matches!(whoami_index(&mut daemon), IndexStatus::Failing { .. })); + assert!(index.holding(&collection()).is_empty()); + + embedder.state.lock().unwrap().down = false; + daemon.index().wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 4, "every admitted document was kept and written"); +} + +/// A collection that is not this mind's is reported as refused, never +/// recreated, and re-read on the backoff: once it is gone the projection +/// completes. +#[test] +fn a_refused_collection_is_reported_and_reread() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + index.plant(&collection(), Described::Unlabelled); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), entries, quick()); + let mut daemon = Daemon::new(mind, sink); + daemon.index().wait_status(|status| matches!(status, IndexStatus::Refused { attempts, .. } if *attempts >= 2)); + assert!(matches!(whoami_index(&mut daemon), IndexStatus::Refused { pending: 4, .. })); + assert!(index.state.lock().unwrap().recreated.is_empty()); + + index.state.lock().unwrap().collections.remove(&collection()); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 4); +} + +// The adapters, against a stub that answers the routes it is given. + +struct Stub { + url: String, + seen: Arc>>, +} + +type Route = (&'static str, u16, &'static str); + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|window| window == needle) +} + +/// Serves each route once, in order, then repeats the last route that matched. +/// A route matches `" "` by prefix. +fn stub(routes: &[Route]) -> Stub { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let seen = Arc::new(Mutex::new(Vec::new())); + let routes: Arc>> = Arc::new(Mutex::new(routes.iter().map(|route| (*route, false)).collect())); + let log = Arc::clone(&seen); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { break }; + let (routes, log) = (Arc::clone(&routes), Arc::clone(&log)); + std::thread::spawn(move || answer_connection(stream, &routes, &log)); + } + }); + Stub { url, seen } +} + +fn answer_connection( + mut stream: TcpStream, + routes: &Mutex>, + log: &Mutex>, +) { + let mut buffer: Vec = Vec::new(); + loop { + let header_end = loop { + if let Some(at) = find(&buffer, b"\r\n\r\n") { + break at + 4; + } + let mut chunk = [0_u8; 4096]; + match stream.read(&mut chunk) { + Ok(0) | Err(_) => return, + Ok(read) => buffer.extend_from_slice(&chunk[..read]), + } + }; + let head = String::from_utf8_lossy(&buffer[..header_end]).into_owned(); + let length = head + .lines() + .find_map(|line| line.to_ascii_lowercase().strip_prefix("content-length:").map(|value| value.trim().parse::().unwrap())) + .unwrap_or(0); + while buffer.len() < header_end + length { + let mut chunk = [0_u8; 4096]; + match stream.read(&mut chunk) { + Ok(0) | Err(_) => return, + Ok(read) => buffer.extend_from_slice(&chunk[..read]), + } + } + let body = String::from_utf8_lossy(&buffer[header_end..header_end + length]).into_owned(); + buffer.drain(..header_end + length); + let mut first = head.lines().next().unwrap().split(' '); + let (method, path) = (first.next().unwrap().to_owned(), first.next().unwrap().to_owned()); + log.lock().unwrap().push((method.clone(), path.clone(), body)); + let key = format!("{method} {path}"); + let (status, answer) = { + let mut routes = routes.lock().unwrap(); + let matching: Vec = + routes.iter().enumerate().filter(|(_, ((prefix, ..), _))| key.starts_with(prefix)).map(|(at, _)| at).collect(); + let chosen = matching.iter().copied().find(|at| !routes[*at].1).or(matching.last().copied()); + match chosen { + Some(at) => { + routes[at].1 = true; + (routes[at].0.1, routes[at].0.2) + } + None => (404, "{}"), + } + }; + let response = + format!("HTTP/1.1 {status} X\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{answer}", answer.len()); + if stream.write_all(response.as_bytes()).is_err() { + return; + } + } +} + +const TAGS: &str = r#"{"models":[{"name":"other:1","model":"other:1","digest":"zzz"},{"name":"m:1","model":"m:1","digest":"ac6d"}]}"#; +const SHOW: &str = r#"{"model_info":{"general.architecture":"qwen3","qwen3.embedding_length":4}}"#; +const EMBED: &str = r#"{"embeddings":[[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0]]}"#; + +#[test] +fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { + let ollama = stub(&[("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]); + let mut embedder = OllamaEmbedder::new(&ollama.url, "m:1"); + assert_eq!(embedder.model_identity().unwrap(), ModelIdentity { name: "m:1".into(), digest: "ac6d".into(), dimensions: 4 }); + let vectors = embedder.embed(&["a".into(), "b".into(), "c".into(), "d".into()]).unwrap(); + assert_eq!(vectors.len(), 4); + assert_eq!(vectors[0], vec![0.5, 1.0, 2.0, 3.0]); + let seen = ollama.seen.lock().unwrap(); + let (_, _, body) = seen.iter().find(|(_, path, _)| path == "/api/embed").unwrap(); + assert!(body.contains("\"model\":\"m:1\"") && body.contains("\"input\":[\"a\",\"b\",\"c\",\"d\"]"), "{body}"); + drop(seen); + + let mut absent = OllamaEmbedder::new(&ollama.url, "not-pulled:1"); + assert!(format!("{:#}", absent.model_identity().unwrap_err()).contains("does not list the model")); + let refusing = stub(&[("POST /api/embed", 500, r#"{"error":"boom"}"#)]); + assert!(OllamaEmbedder::new(&refusing.url, "m:1").embed(&["a".into()]).is_err()); +} + +/// Compatibility never reads the address: the same model at another address +/// leaves the collection alone, and the same address serving another digest +/// rebuilds it. +#[test] +fn the_embedders_address_is_not_part_of_the_collections_compatibility() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let routes = [("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]; + let repulled = TAGS.replace("ac6d", "beef"); + let index = FakeIndex::default(); + + let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE)); + first.want(entries.iter().cloned()); + drain(&mut first); + assert_eq!(counts(&index), (1, 1)); + + // Another address, the same model and digest. + let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE)); + moved.want(entries.iter().cloned()); + drain(&mut moved); + assert_eq!(counts(&index), (1, 1)); + + // The same model name re-pulled: another digest. + let leaked: &'static str = Box::leak(repulled.into_boxed_str()); + let other = [("GET /api/tags", 200, leaked), routes[1], routes[2]]; + let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE)); + rebuilt.want(entries.iter().cloned()); + drain(&mut rebuilt); + assert_eq!(counts(&index), (2, 2)); +} + +#[test] +fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { + let name = collection(); + let absent = stub(&[("GET /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); + assert_eq!(QdrantIndex::new(&absent.url).describe(&name).unwrap(), Described::Absent); + + let unlabelled = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":null}}}"#)]); + assert_eq!(QdrantIndex::new(&unlabelled.url).describe(&name).unwrap(), Described::Unlabelled); + let voidbot = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":{"managedBy":"voidbot"}}}}"#)]); + assert_eq!(QdrantIndex::new(&voidbot.url).describe(&name).unwrap(), Described::Unlabelled); + + let labelled = stub(&[( + "GET /collections/", + 200, + r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, + )]); + assert_eq!( + QdrantIndex::new(&labelled.url).describe(&name).unwrap(), + Described::Labelled(CollectionMeta { + managed_by: "huginn".into(), + instance: "yggdrasil".into(), + model: "m:1".into(), + model_digest: "ac6d".into(), + dimensions: 4, + index_text_version: 1, + }) + ); + + // A scroll is followed to its last page. + let paged = stub(&[ + ("POST /collections/", 200, r#"{"result":{"points":[{"id":"a"},{"id":"b"}],"next_page_offset":"b"}}"#), + ("POST /collections/", 200, r#"{"result":{"points":[{"id":"c"}],"next_page_offset":null}}"#), + ]); + let ids = QdrantIndex::new(&paged.url).ids(&name).unwrap(); + let expected: BTreeSet = ["a", "b", "c"].map(String::from).into_iter().collect(); + assert_eq!(ids, expected); + let seen = paged.seen.lock().unwrap(); + assert!(!seen[0].2.contains("offset") && seen[1].2.contains("\"offset\":\"b\""), "{seen:?}"); + drop(seen); + + // A rebuild drops, then creates with the metadata and the vector shape. + let writing = stub(&[("DELETE /collections/", 200, "{}"), ("PUT /collections/", 200, "{}")]); + let meta = CollectionMeta { + managed_by: MANAGED_BY.into(), + instance: INSTANCE.into(), + model: "m:1".into(), + model_digest: "ac6d".into(), + dimensions: 4, + index_text_version: INDEX_TEXT_VERSION, + }; + let mut store = QdrantIndex::new(&writing.url); + store.recreate(&name, &meta).unwrap(); + let point = Point { + id: point_id("d"), + vector: vec![1.0, 2.0, 3.0, 4.0], + payload: PointPayload { doc_id: "d".into(), kind: "question".into(), root: "eureka-state".into(), ordinal: 7, text_sha256: "h".into() }, + }; + store.upsert(&name, &[point]).unwrap(); + let seen = writing.seen.lock().unwrap(); + let paths: Vec = seen.iter().map(|(method, path, _)| format!("{method} {path}")).collect(); + assert_eq!( + paths, + [ + format!("DELETE /collections/{name}?wait=true"), + format!("PUT /collections/{name}?wait=true"), + format!("PUT /collections/{name}/points?wait=true"), + ] + ); + for expected in ["\"size\":4", "\"distance\":\"Cosine\"", "\"on_disk\":true", "\"managed_by\":\"huginn\"", "\"index_text_version\":1"] { + assert!(seen[1].2.contains(expected), "{expected} in {}", seen[1].2); + } + for expected in ["\"doc_id\":\"d\"", "\"kind\":\"question\"", "\"root\":\"eureka-state\"", "\"ordinal\":7", "\"text_sha256\":\"h\""] { + assert!(seen[2].2.contains(expected), "{expected} in {}", seen[2].2); + } + + let failing = stub(&[("PUT /collections/", 500, r#"{"status":{"error":"disk full"}}"#)]); + assert!(QdrantIndex::new(&failing.url).upsert(&name, &[]).is_err()); +} diff --git a/crates/huginn-daemon/src/lib.rs b/crates/huginn-daemon/src/lib.rs index ee36d7c..01e9d9a 100644 --- a/crates/huginn-daemon/src/lib.rs +++ b/crates/huginn-daemon/src/lib.rs @@ -1,8 +1,9 @@ //! `huginn-daemon`: one process that serves one mind over CultNet. //! -//! The crate owns four things and no rule. `Daemon::open` owns "may this -//! process serve this mind", by delegating to `Mind::open` and refusing before -//! anything listens. `Daemon::handle` owns "which `Mind` method answers this +//! The crate owns five things and no rule. `startup` owns "may this process +//! serve this mind", by delegating to `Mind::open` and refusing before +//! anything listens. `index` owns when the mind's documents are embedded and +//! written to the vector store, off the serving path. `Daemon::handle` owns "which `Mind` method answers this //! request": it dispatches, it asks `Mind::require_instance` for every read //! that names an instance, and it compares nothing itself. `envelope` owns "is //! this a request at all": a message it cannot decode is answered with a typed @@ -14,8 +15,9 @@ pub mod daemon; pub mod envelope; +pub mod index; pub mod serve; -pub use daemon::{Daemon, IndexSink, NoIndex}; +pub use daemon::{Daemon, IndexSink, runtime_id}; pub use envelope::{FAILURE_SCHEMA, OperationFailure}; pub use serve::{Options, ServeOptions, parse_options, run, startup}; diff --git a/crates/huginn-daemon/src/main.rs b/crates/huginn-daemon/src/main.rs index 2c14b8d..ce28fb3 100644 --- a/crates/huginn-daemon/src/main.rs +++ b/crates/huginn-daemon/src/main.rs @@ -6,11 +6,16 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use anyhow::{Context, Result}; +use huginn_daemon::index::Backoff; +use huginn_daemon::index::ollama::OllamaEmbedder; +use huginn_daemon::index::qdrant::QdrantIndex; use huginn_daemon::serve::{ServeOptions, parse_options, run, startup}; fn main() -> Result<()> { let options = parse_options(std::env::args().skip(1))?; - let (mut daemon, mut hub, registry) = match startup(&options) { + let embedder = OllamaEmbedder::new(&options.ollama_url, &options.embedding_model); + let index = QdrantIndex::new(&options.qdrant_url); + let (mut daemon, mut hub, registry) = match startup(&options, embedder, index, Backoff::default()) { Ok(started) => started, Err(error) => { eprintln!("huginn-daemon refuses to serve {}: {error:#}", options.instance.0); diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index 888d5a2..803b9ce 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -61,9 +61,10 @@ use huginn_mind::wire::{ HuginnMindResponse, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON, MIND_RESPONSE_SCHEMA, MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, }; -use huginn_mind::{MindRefusal, MindStore, OwnedRedbMessagePackBackingStore}; +use huginn_mind::{Mind, MindRefusal, MindStore, OwnedRedbMessagePackBackingStore}; -use crate::daemon::{Daemon, IndexSink, NoIndex}; +use crate::daemon::{Daemon, IndexSink, runtime_id}; +use crate::index::{Backoff, Embedder, VectorIndex, WorkerSink}; use crate::envelope::{decode_request, encode_failure, encode_response}; /// What the loop does when nothing is waiting. @@ -78,22 +79,32 @@ impl Default for ServeOptions { } } -/// The three things the operator supplies. Nothing here reads the environment: -/// how Idunn supplies `--bind` is Cut 14's. +/// The six things the operator supplies. Nothing here reads the environment: +/// how Idunn supplies `--bind` is Cut 14's. The index has no off switch: an +/// unreachable Qdrant or Ollama is a degraded projection, reported by +/// `whoami`, not a configuration. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Options { pub state_root: PathBuf, pub instance: Slug, pub bind: SocketAddr, + pub qdrant_url: String, + pub ollama_url: String, + pub embedding_model: String, } -/// Exactly `--state-root`, `--instance` and `--bind`, each required, each once. +/// Exactly `--state-root`, `--instance`, `--bind`, `--qdrant-url`, +/// `--ollama-url` and `--embedding-model`, each required, each once. The two +/// URLs are plain `http://`: this build carries no TLS. pub fn parse_options(args: impl Iterator) -> Result { let mut args = args; let mut values: BTreeMap = BTreeMap::new(); while let Some(name) = args.next() { let name = name.strip_prefix("--").with_context(|| format!("expected --option, got {name:?}"))?; - ensure!(matches!(name, "state-root" | "instance" | "bind"), "unsupported Huginn option --{name}"); + ensure!( + matches!(name, "state-root" | "instance" | "bind" | "qdrant-url" | "ollama-url" | "embedding-model"), + "unsupported Huginn option --{name}" + ); let value = args.next().with_context(|| format!("missing value for --{name}"))?; ensure!(values.insert(name.to_owned(), value).is_none(), "duplicate Huginn option --{name}"); } @@ -103,22 +114,44 @@ pub fn parse_options(args: impl Iterator) -> Result { let state_root = PathBuf::from(take("state-root")?); ensure!(state_root.is_absolute(), "--state-root must be an absolute path"); let bind = take("bind")?; + let url = |name: &str| -> Result { + let value = take(name)?; + ensure!(value.starts_with("http://"), "--{name} must be a plain http:// address, got {value:?}"); + Ok(value) + }; + let embedding_model = take("embedding-model")?; + ensure!(!embedding_model.is_empty(), "--embedding-model must name a model"); Ok(Options { state_root, instance: Slug(take("instance")?), bind: bind.parse().with_context(|| format!("--bind must be an ip:port, got {bind:?}"))?, + qdrant_url: url("qdrant-url")?, + ollama_url: url("ollama-url")?, + embedding_model, }) } /// Ruling 15, as an order: the mind opens first, and only a mind that opened -/// gets a socket. This is the one place both happen, so nothing can bind ahead -/// of the refusal. -pub fn startup( +/// gets a socket, and only a daemon that is going to serve gets an index +/// worker. This is the one place all three happen, so nothing can bind ahead +/// of the refusal and nothing reaches the vector store for a process that +/// never listens. The worker's first act is to reconcile the collection +/// against every indexable document the mind holds now. +pub fn startup( options: &Options, -) -> Result<(Daemon, CultNetRudpServerHub, CultNetSchemaRegistry)> { - let daemon = Daemon::open(&options.state_root, &options.instance)?; - let hub = bind(options.bind, &daemon.runtime_id())?; - Ok((daemon, hub, schema_registry()?)) + embedder: E, + index: V, + backoff: Backoff, +) -> Result<(Daemon, CultNetRudpServerHub, CultNetSchemaRegistry)> +where + E: Embedder + Send + 'static, + V: VectorIndex + Send + 'static, +{ + let mind = Mind::open(&options.state_root, &options.instance)?; + let hub = bind(options.bind, &runtime_id(&options.instance))?; + let startup_entries = mind.index_entries(None)?; + let sink = WorkerSink::spawn(embedder, index, &options.instance, startup_entries, backoff); + Ok((Daemon::new(mind, sink), hub, schema_registry()?)) } /// The bytes one reliable packet carries, and the packets one session may hold @@ -310,10 +343,12 @@ pub fn run>( mod tests { use super::*; use crate::daemon::tests::{ - FITTING_CHANGES, FITTING_CUT, INSTANCE, OTHER, WIDE_CHANGES, WIDE_CUT, batch, identity, now, seeded_wide, slug, + FITTING_CHANGES, FITTING_CUT, INSTANCE, NoIndex, OTHER, WIDE_CHANGES, WIDE_CUT, batch, identity, now, + open_unindexed, seeded_wide, slug, }; use huginn_mind::epiphany_pipeline::{PipelineKind, PipelineRef}; use crate::envelope::{FAILURE_SCHEMA, decode_response, encode_request}; + use crate::index::Backoff; use base64::Engine; use base64::engine::general_purpose::STANDARD; use cultnet_rs::{CultMesh, CultMeshRudpSocketOptions}; @@ -365,7 +400,7 @@ mod tests { #[test] fn a_malformed_envelope_is_answered_with_a_failure_and_touches_no_mind() { let root = tempfile::tempdir().unwrap(); - let mut daemon = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let mut daemon = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); let registry = schema_registry().unwrap(); let admit = STANDARD.encode( rmp_serde::to_vec_named(&HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)]))).unwrap(), @@ -446,7 +481,7 @@ mod tests { #[test] fn two_clients_get_their_own_replies_over_loopback() { let root = tempfile::tempdir().unwrap(); - let mut daemon = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let mut daemon = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); let mut hub = bind("127.0.0.1:0".parse().unwrap(), &daemon.runtime_id()).unwrap(); let endpoint = format!("rudp://{}", hub.local_addr().unwrap()); let registry = schema_registry().unwrap(); @@ -944,46 +979,76 @@ mod tests { #[test] fn startup_opens_the_mind_before_it_binds_anything() { let root = tempfile::tempdir().unwrap(); - let held = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let held = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); let occupied = UdpSocket::bind("127.0.0.1:0").unwrap(); let port = occupied.local_addr().unwrap(); - let options = Options { state_root: root.path().to_path_buf(), instance: slug(INSTANCE), bind: port }; - let error = format!("{:#}", startup(&options).err().expect("a held mind refuses")); + let options = Options { + state_root: root.path().to_path_buf(), + instance: slug(INSTANCE), + bind: port, + qdrant_url: "http://127.0.0.1:1".into(), + ollama_url: "http://127.0.0.1:1".into(), + embedding_model: "a-model".into(), + }; + let started = |options: &Options| { + let (embedder, index) = crate::index::fakes::pair(); + startup(options, embedder, index, Backoff::default()) + }; + let error = format!("{:#}", started(&options).err().expect("a held mind refuses")); assert!(error.contains("MindAlreadyOwned"), "the refusal is the mind's, not the socket's: {error}"); assert!(!error.contains("binding"), "the socket was never reached: {error}"); // With the mind free and the port still held, the socket is the only // gate left, and startup reports it. drop(held); - let error = format!("{:#}", startup(&options).err().expect("a held port refuses")); + let error = format!("{:#}", started(&options).err().expect("a held port refuses")); assert!(error.contains("binding"), "{error}"); drop(occupied); - let (_daemon, hub, _registry) = startup(&options).expect("both gates open"); + let (_daemon, hub, _registry) = started(&options).expect("both gates open"); assert_eq!(hub.local_addr().unwrap(), port); } #[test] - fn options_are_exactly_state_root_instance_and_bind() { + fn options_are_exactly_the_six_the_operator_supplies() { let root = if cfg!(windows) { r"C:\state" } else { "/state" }; let args = |values: &[&str]| values.iter().map(|value| value.to_string()).collect::>().into_iter(); + let good = [ + "--state-root", root, "--instance", "yggdrasil", "--bind", "127.0.0.1:17872", "--qdrant-url", + "http://127.0.0.1:6333", "--ollama-url", "http://10.77.0.4:11434", "--embedding-model", "qwen3-embedding:0.6b", + ]; assert_eq!( - parse_options(args(&["--state-root", root, "--instance", "yggdrasil", "--bind", "127.0.0.1:17872"])) - .unwrap(), + parse_options(args(&good)).unwrap(), Options { state_root: PathBuf::from(root), instance: slug("yggdrasil"), bind: "127.0.0.1:17872".parse().unwrap(), + qdrant_url: "http://127.0.0.1:6333".into(), + ollama_url: "http://10.77.0.4:11434".into(), + embedding_model: "qwen3-embedding:0.6b".into(), } ); + // Each of the six is required: there is no index-less mode. + for name in ["--state-root", "--instance", "--bind", "--qdrant-url", "--ollama-url", "--embedding-model"] { + let at = good.iter().position(|value| *value == name).unwrap(); + let mut without = good.to_vec(); + without.drain(at..at + 2); + assert!(parse_options(args(&without)).is_err(), "{name} is required"); + } + fn with<'a>(good: &[&'a str], name: &str, value: &'a str) -> Vec<&'a str> { + let at = good.iter().position(|option| *option == name).unwrap(); + let mut changed = good.to_vec(); + changed[at + 1] = value; + changed + } for bad in [ - vec!["--state-root", root, "--instance", "yggdrasil"], - vec!["--state-root", root, "--bind", "127.0.0.1:1"], - vec!["--instance", "yggdrasil", "--bind", "127.0.0.1:1"], - vec!["--state-root", root, "--instance", "yggdrasil", "--bind", "127.0.0.1:1", "--bind", "127.0.0.1:2"], - vec!["--state-root", root, "--instance", "yggdrasil", "--bind", "not-an-address"], - vec!["--state-root", "relative", "--instance", "yggdrasil", "--bind", "127.0.0.1:1"], - vec!["--state-root", root, "--instance", "yggdrasil", "--bind", "127.0.0.1:1", "--idunn-anchor", "x"], + [good.to_vec(), vec!["--bind", "127.0.0.1:2"]].concat(), + with(&good, "--bind", "not-an-address"), + with(&good, "--state-root", "relative"), + with(&good, "--qdrant-url", "https://127.0.0.1:6333"), + with(&good, "--ollama-url", "10.77.0.4:11434"), + with(&good, "--embedding-model", ""), + [good.to_vec(), vec!["--idunn-anchor", "x"]].concat(), ] { assert!(parse_options(args(&bad)).is_err(), "{bad:?}"); } diff --git a/crates/huginn-mind/src/index.rs b/crates/huginn-mind/src/index.rs new file mode 100644 index 0000000..7ac4869 --- /dev/null +++ b/crates/huginn-mind/src/index.rs @@ -0,0 +1,312 @@ +//! What of a document is searchable, and nothing else about the index. +//! +//! `index_text` is a rule about the leaf's documents: which fields are prose a +//! reader would search for. It is a pure function of one document, so it reads +//! no receipt, no provenance and no status, and nothing derived from it is +//! stored: the daemon's projection recomputes it, and `INDEX_TEXT_VERSION` is +//! how a change to it makes every projection rebuild. No network, no JSON, no +//! store handle: the daemon owns the vector store and the embedder. + +use epiphany_pipeline::{PipelineDocument, PipelineKind, PipelineRef}; +use sha2::{Digest, Sha256}; + +use crate::docs::Docs; +use crate::mind::Mind; +use crate::query::AdmissionIndex; +use crate::refusal::MindRefusal; +use crate::rows::root_and_local; +use crate::store::MindStore; + +/// Bumped whenever `index_text` changes what it returns for any document. +pub const INDEX_TEXT_VERSION: u32 = 1; + +/// The most bytes one document's text may carry, cut at a character boundary. +pub const INDEX_TEXT_MAX_BYTES: usize = 16 * 1024; + +/// One document as the projection needs it: the key, the searchable text and +/// the digest of that text, and the ordinal of the receipt that wrote it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IndexEntry { + pub id: PipelineRef, + /// The key's root segment: a campaign slug, or an instance. + pub root: String, + pub text: String, + pub text_sha256: String, + pub ordinal: u64, +} + +/// The searchable text of a document, or `None` when its kind carries none +/// (`instance`, `stewardship`) or every field it names is empty. Lines are +/// joined by newlines. Withdrawn resolutions are indexed like any other: a +/// closed thing stays findable. +pub fn index_text(document: &PipelineDocument) -> Option { + use PipelineDocument as D; + let mut lines: Vec<&str> = Vec::new(); + match document { + D::Campaign(campaign) => lines.push(&campaign.title.0), + D::Target(target) => { + lines.extend(target.invariants.iter().map(|invariant| invariant.statement.0.as_str())); + lines.extend(target.not_in_scope.iter().map(|line| line.0.as_str())); + lines.extend(target.canonical_implementations.iter().map(|line| line.0.as_str())); + } + D::Question(question) => { + lines.push(&question.title.0); + lines.push(&question.question.0); + lines.extend(question.options.iter().map(|option| option.text.0.as_str())); + } + D::Ruling(ruling) => { + lines.push(&ruling.title.0); + lines.push(&ruling.ruling.0); + lines.extend(ruling.operator_quote.iter().map(|quote| quote.0.as_str())); + } + D::CutSpec(spec) => { + lines.push(&spec.title.0); + lines.extend(spec.first.iter().map(|line| line.0.as_str())); + lines.extend(spec.adds.iter().map(|line| line.0.as_str())); + lines.extend(spec.keeps_moves.iter().map(|line| line.0.as_str())); + } + D::CutReport(report) => { + lines.extend(report.commits.iter().map(|commit| commit.subject.0.as_str())); + for deviation in &report.deviations { + lines.push(&deviation.what.0); + lines.push(&deviation.why.0); + } + lines.extend(report.undone.iter().map(|line| line.0.as_str())); + lines.extend(report.promises.iter().map(|promise| promise.text.0.as_str())); + } + D::Verdict(verdict) => lines.extend(verdict.claims.iter().map(|claim| claim.claim.0.as_str())), + D::Finding(finding) => { + lines.push(&finding.claim.0); + lines.push(&finding.failure_scenario.0); + lines.extend(finding.evidence.iter().map(|evidence| evidence.result.0.as_str())); + } + D::FollowUp(follow_up) => { + lines.push(&follow_up.item.0); + lines.push(&follow_up.why_it_can_wait.0); + } + D::Resolution(resolution) => lines.push(&resolution.rationale.0), + D::HandOff(hand_off) => lines.push(&hand_off.reason.0), + D::Instance(_) | D::Stewardship(_) => {} + } + lines.retain(|line| !line.is_empty()); + if lines.is_empty() { + return None; + } + let mut text = lines.join("\n"); + if text.len() > INDEX_TEXT_MAX_BYTES { + let mut cut = INDEX_TEXT_MAX_BYTES; + while !text.is_char_boundary(cut) { + cut -= 1; + } + text.truncate(cut); + } + Some(text) +} + +fn sha256_hex(text: &str) -> String { + Sha256::digest(text.as_bytes()).iter().map(|byte| format!("{byte:02x}")).collect() +} + +impl Mind { + /// The entries to project: every indexable document the mind holds, or + /// only the named ones. The ordinal is the writing receipt's. A named + /// document the mind does not hold is a refusal, not a silent omission: a + /// caller names what a commit just landed. + pub fn index_entries(&self, only: Option<&[PipelineRef]>) -> Result, MindRefusal> { + let documents: Vec<(PipelineKind, String, PipelineDocument)> = match only { + Some(refs) => { + let mut held = Vec::with_capacity(refs.len()); + for reference in refs { + reference.validate_ref()?; + let document = self.get(reference.kind, &reference.id.0)?.ok_or_else(|| { + MindRefusal::Unavailable { detail: format!("index: {} is not held", reference.id.0) } + })?; + held.push((reference.kind, reference.id.0.clone(), document)); + } + held + } + None => Docs::from_image(self.envelopes())? + .image + .into_iter() + .map(|held| (held.kind, held.key, held.document)) + .collect(), + }; + let facts = AdmissionIndex::build(self)?; + let mut entries = Vec::new(); + for (kind, key, document) in documents { + let Some(text) = index_text(&document) else { continue }; + let ordinal = facts.ordinal_of(kind, &key)?; + entries.push(IndexEntry { + root: root_and_local(&key).0.to_string(), + id: PipelineRef { kind, id: key.as_str().into() }, + text_sha256: sha256_hex(&text), + text, + ordinal, + }); + } + Ok(entries) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixtures::*; + use crate::receipt::Faculty; + use crate::store::test_stores::MemoryStore; + use epiphany_pipeline::{ + ClaimOutcome, Deviation, FindingConfidence, PipelineDocument as D, PipelineKind as K, RulingAuthority, + }; + use std::collections::BTreeMap; + + /// A mind holding one document of every kind that carries text, each with + /// more than one text field where the kind has more than one, and the + /// question's closure withdrawn beside them. Returns the hand-off's key. + fn full_mind() -> (Mind, String) { + let mut mind = opened(MemoryStore::new(), INSTANCE); + let D::Target(mut target_doc) = target(1, &[INVARIANT, "second"]) else { panic!() }; + target_doc.not_in_scope = vec!["No client writes.".into()]; + target_doc.canonical_implementations = vec!["huginn-mind admission".into()]; + committed(admit( + &mut mind, + vec![instance(INSTANCE), stewardship(INSTANCE, REPO), campaign(&[REPO]), D::Target(target_doc)], + )); + committed(admit(&mut mind, vec![question("Q1", &["A", "B"], "A")])); + let mut ruling_doc = ruling("R1"); + ruling_doc.answers = Some(s(&id("question", "Q1"))); + ruling_doc.choice = Some(l("A")); + ruling_doc.operator_quote = Some("Own it.".into()); + ruling_doc.authority = RulingAuthority::Operator; + committed(admit(&mut mind, vec![D::Ruling(ruling_doc)])); + // The derived closure of Q1 is withdrawn: a withdrawn resolution is a + // document like any other and stays findable. + let closed = id("resolution", "question.Q1.n1"); + committed(admit(&mut mind, vec![resolution(r(K::Resolution, &closed), withdrawn())])); + let mut spec_doc = cut_spec("1", 1); + spec_doc.first = vec!["Read the leaf.".into()]; + spec_doc.adds = vec!["An index module.".into(), "A worker.".into()]; + spec_doc.keeps_moves = vec!["Admission stays.".into()]; + committed(admit(&mut mind, vec![D::CutSpec(spec_doc)])); + let mut report_doc = cut_report("1", 1); + report_doc.deviations = vec![Deviation { what: "Changed a name.".into(), why: "It collided.".into() }]; + report_doc.undone = vec!["The live smoke.".into()]; + committed(admit( + &mut mind, + vec![D::CutReport(report_doc), follow_up("FU-1", r(K::Question, &id("question", "Q1")))], + )); + committed(admit( + &mut mind, + vec![ + verdict("1", 1, vec![claim(ClaimOutcome::Falsified, &[&id("finding", "cut-1.s1.F1")], Some("P1"), &["M1"])]), + D::Finding(finding("1", 1, "F1", FindingConfidence::Confirmed)), + ], + )); + let (_, writes) = committed(admit(&mut mind, vec![hand_off(INSTANCE, OTHER_INSTANCE, REPO, &[])])); + let hand_off_key = writes.iter().find(|write| write.kind == K::HandOff).unwrap().id.0.clone(); + (mind, hand_off_key) + } + + fn texts(mind: &Mind) -> BTreeMap { + mind.index_entries(None).unwrap().into_iter().map(|entry| (entry.id.id.0, entry.text)).collect() + } + + /// Each kind's search text is its named fields, joined by newlines, and + /// the two kinds with no prose have no entry. + #[test] + fn index_text_is_each_kinds_named_fields_and_the_identity_kinds_have_none() { + let (mind, hand_off_key) = full_mind(); + let expected: Vec<(String, &str)> = vec![ + (id("campaign", "self"), "Eureka pipeline state"), + (id("target", "r1"), "Only admission writes.\nOnly admission writes.\nNo client writes.\nhuginn-mind admission"), + (id("question", "Q1"), "Who owns the state?\nWho owns the state?\nan option\nan option"), + (id("ruling", "R1"), "An instance owns its mind\nAn instance owns its mind.\nOwn it."), + (id("cut_spec", "cut-1.r1"), "A cut\nRead the leaf.\nAn index module.\nA worker.\nAdmission stays."), + ( + id("cut_report", "cut-1.h1"), + "Land the cut\nChanged a name.\nIt collided.\nThe live smoke.\nOne derived key per document.", + ), + (id("verdict", "cut-1.s1"), "Falsified claim"), + (id("finding", "cut-1.s1.F1"), "A dotted label composes two keys.\nTwo documents claim one key.\nok"), + (id("follow_up", "FU-1"), "Per-kind admission rules.\nThe organ owns admission."), + // The withdrawn closure of Q1, admitted by hand. + (id("resolution", "resolution.question.Q1.n1.n1"), "Resolved."), + (hand_off_key, "The workstation mind takes the campaign."), + ]; + let held = texts(&mind); + for (key, text) in &expected { + match held.get(key) { + Some(found) => assert_eq!(found, text, "{key}"), + None => panic!("{key} has no entry; entries are {:?}", held.keys().collect::>()), + } + } + let kinds: Vec = mind.index_entries(None).unwrap().iter().map(|entry| entry.id.kind).collect(); + assert!(!kinds.contains(&K::Instance) && !kinds.contains(&K::Stewardship), "{kinds:?}"); + assert!(kinds.contains(&K::Resolution), "the derived closures are indexed too"); + } + + /// The text is the document's alone: the same document admitted by another + /// faculty at another clock, or closed by a later ruling, has the same + /// text and digest. Only the ordinal is the receipt's. + #[test] + fn index_text_ignores_admission_facts_and_indexes_withdrawn_resolutions() { + let mut mind = seeded(); + committed(admit(&mut mind, vec![question("Q1", &["A", "B"], "A")])); + let q1 = id("question", "Q1"); + let before = mind.index_entries(Some(&[r(K::Question, &q1)])).unwrap(); + + let mut other = seeded(); + committed(admit_as(&mut other, Faculty::Soul, vec![question("Q1", &["A", "B"], "A")])); + let mut later = ruling("R1"); + later.answers = Some(s(&q1)); + later.choice = Some(l("A")); + committed(admit(&mut other, vec![D::Ruling(later)])); + let after = other.index_entries(Some(&[r(K::Question, &q1)])).unwrap(); + assert_eq!(before, after, "a closed question is the same entry as an open one"); + assert_eq!(before[0].ordinal, 2); + + let closure = id("resolution", "question.Q1.n1"); + committed(admit(&mut other, vec![resolution(r(K::Resolution, &closure), withdrawn())])); + let withdrawn_id = id("resolution", "resolution.question.Q1.n1.n1"); + let entry = other.index_entries(Some(&[r(K::Resolution, &withdrawn_id)])).unwrap(); + assert_eq!(entry.len(), 1, "a withdrawn resolution is indexed"); + assert_eq!(entry[0].text, "Resolved."); + assert_eq!(entry[0].root, CAMPAIGN); + assert_eq!(entry[0].ordinal, 4); + } + + #[test] + fn the_digest_is_sha256_of_the_text_and_a_missing_document_is_refused() { + let mind = seeded(); + let campaign_ref = r(K::Campaign, &id("campaign", "self")); + let entry = &mind.index_entries(Some(std::slice::from_ref(&campaign_ref))).unwrap()[0]; + assert_eq!(entry.text_sha256, "49c7e65adaad4bd34261300e9c5575847357b8f269508c9b2a46ec2aa75cc019"); + assert_eq!((entry.root.as_str(), entry.ordinal), (CAMPAIGN, 1)); + let absent = r(K::Question, &id("question", "Q9")); + assert!(matches!(mind.index_entries(Some(&[absent])), Err(MindRefusal::Unavailable { .. }))); + // The identity and the stewardship carry no text: the seeded mind's + // indexable documents are its campaign and its target. + let all = mind.index_entries(None).unwrap(); + assert_eq!(all.len(), 2, "{all:?}"); + } + + /// The bound is bytes, cut at a character boundary: three-byte characters + /// put the byte limit inside one. + #[test] + fn text_is_bounded_at_a_character_boundary() { + let wide: String = format!("{}x", "\u{20ac}".repeat(333)); + assert_eq!(wide.len(), 1000); + let mut spec = cut_spec("1", 1); + spec.adds = (0..40).map(|_| wide.as_str().into()).collect(); + let text = index_text(&D::CutSpec(spec)).unwrap(); + // The title and its newline, then full lines of 1,001 bytes, then part + // of the next one, cut back to the last whole character. + let head = "A cut\n".len(); + let stride = 1001; + let full_lines = (INDEX_TEXT_MAX_BYTES - head) / stride; + let consumed = head + full_lines * stride; + let inside = INDEX_TEXT_MAX_BYTES - consumed; + assert_ne!(inside % 3, 0, "the limit falls inside a character"); + assert_eq!(text.len(), consumed + inside - inside % 3); + assert!(text.ends_with('\u{20ac}')); + } +} diff --git a/crates/huginn-mind/src/lib.rs b/crates/huginn-mind/src/lib.rs index 14c5d1f..c402f4c 100644 --- a/crates/huginn-mind/src/lib.rs +++ b/crates/huginn-mind/src/lib.rs @@ -26,6 +26,7 @@ pub mod admission; mod docs; +pub mod index; pub mod mind; mod rows; pub mod query; @@ -38,6 +39,7 @@ pub mod wire; pub(crate) mod fixtures; pub use docs::CitationRole; +pub use index::{INDEX_TEXT_MAX_BYTES, INDEX_TEXT_VERSION, IndexEntry, index_text}; pub use admission::{BATCH_MAX, PipelineAdmissionBatch, PipelineAdmissionOutcome}; pub use mind::{HuginnMindEpoch, Mind}; pub use query::{ @@ -49,7 +51,7 @@ pub use refusal::MindRefusal; pub use store::{MindStore, OwnedRedbMessagePackBackingStore}; pub use wire::{ HuginnMindRequest, HuginnMindResponse, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON, MIND_RESPONSE_SCHEMA, - MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, MindStatus, + MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, IndexStatus, MindStatus, }; /// The leaf, whole, through the one crate that pins its rev: the daemon and diff --git a/crates/huginn-mind/src/query.rs b/crates/huginn-mind/src/query.rs index e657509..a85e353 100644 --- a/crates/huginn-mind/src/query.rs +++ b/crates/huginn-mind/src/query.rs @@ -16,8 +16,8 @@ //! over the documents admitted at or before the page's `asOf`), the //! projection to a header or a whole view, and the typing of the edges. //! -//! `semantic` is accepted by the type and refused typed until Cut 11 wires the -//! index. No clock, no store handle, no network. +//! `semantic` is accepted by the type and refused typed until Cut 11b wires the +//! read side of the index. No clock, no store handle, no network. use std::collections::BTreeMap; @@ -304,11 +304,21 @@ impl AdmissionIndex { } fn of(&self, held: &Held) -> Result { - let type_id = held.kind.type_id(); + self.facts_of(held.kind, &held.key) + } + + fn facts_of(&self, kind: PipelineKind, key: &str) -> Result { + let type_id = kind.type_id(); self.0 - .get(&(type_id.to_string(), held.key.clone())) + .get(&(type_id.to_string(), key.to_string())) .cloned() - .ok_or_else(|| integrity(type_id, &held.key, "has no commit receipt")) + .ok_or_else(|| integrity(type_id, key, "has no commit receipt")) + } + + /// The ordinal of the receipt that wrote a document: the projection's + /// position for it. + pub(crate) fn ordinal_of(&self, kind: PipelineKind, key: &str) -> Result { + self.facts_of(kind, key).map(|facts| facts.ordinal) } } diff --git a/crates/huginn-mind/src/wire.rs b/crates/huginn-mind/src/wire.rs index d39749e..ee825d3 100644 --- a/crates/huginn-mind/src/wire.rs +++ b/crates/huginn-mind/src/wire.rs @@ -106,6 +106,26 @@ impl HuginnMindResponse { } } +/// How far the semantic index has caught up with the mind. The mind decides +/// none of it: the daemon's index worker reports it, in memory, and a restart +/// recomputes it. `pending` counts documents not yet known to be in the index. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +pub enum IndexStatus { + /// The worker has not yet compared the mind with the collection. + Reconciling { pending: u32 }, + /// Every indexable document is in the collection. + Current, + /// Reconciled, and indexing what is still pending. + Behind { pending: u32 }, + /// The last attempt failed (the embedder or the vector store is + /// unreachable, or refused); it is retried with a backoff and nothing is + /// dropped. `attempts` counts consecutive failures. + Failing { pending: u32, attempts: u32, error: String }, + /// The collection belongs to another writer. The worker never recreates + /// it; it re-reads it on the same backoff. + Refused { pending: u32, attempts: u32, reason: String }, +} + /// What a mind says about itself: the typed state a dashboard projects. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct MindStatus { @@ -115,12 +135,13 @@ pub struct MindStatus { /// not documents and are not counted here. pub documents: u32, pub receipts: u32, + pub index: IndexStatus, } impl Mind { - /// `whoami`, derived from the image every time it is asked. Nothing stores - /// a status and no configuration supplies one. - pub fn status(&self) -> MindStatus { + /// `whoami`, derived from the image every time it is asked, and from the + /// index status the caller holds: the mind stores neither. + pub fn status(&self, index: IndexStatus) -> MindStatus { let mut documents = 0_u32; let mut receipts = 0_u32; for envelope in self.envelopes() { @@ -135,6 +156,7 @@ impl Mind { schema_epoch: PIPELINE_SCHEMA_EPOCH.to_string(), documents, receipts, + index, } } } @@ -163,7 +185,7 @@ mod tests { fn responses() -> Vec { let refusal = MindRefusal::ForeignInstance { declared: OTHER_INSTANCE.into(), mind: INSTANCE.into() }; vec![ - HuginnMindResponse::Whoami(seeded().status()), + HuginnMindResponse::Whoami(seeded().status(IndexStatus::Current)), HuginnMindResponse::Admit(PipelineAdmissionOutcome::AlreadyAdmitted { receipt_id: "r1".into() }), HuginnMindResponse::Admit(PipelineAdmissionOutcome::Conflict { identities: vec![] }), HuginnMindResponse::Admit(PipelineAdmissionOutcome::Refused(refusal.clone())), diff --git a/schemas/cultnet/huginn.mind_response.v1.schema.json b/schemas/cultnet/huginn.mind_response.v1.schema.json index 39cc870..d30c2ca 100644 --- a/schemas/cultnet/huginn.mind_response.v1.schema.json +++ b/schemas/cultnet/huginn.mind_response.v1.schema.json @@ -443,6 +443,128 @@ "type": "string", "maxLength": 40 }, + "IndexStatus": { + "description": "How far the semantic index has caught up with the mind. The mind decides\nnone of it: the daemon's index worker reports it, in memory, and a restart\nrecomputes it. `pending` counts documents not yet known to be in the index.", + "oneOf": [ + { + "description": "The worker has not yet compared the mind with the collection.", + "type": "object", + "properties": { + "Reconciling": { + "type": "object", + "properties": { + "pending": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "pending" + ] + } + }, + "additionalProperties": false, + "required": [ + "Reconciling" + ] + }, + { + "description": "Every indexable document is in the collection.", + "type": "string", + "const": "Current" + }, + { + "description": "Reconciled, and indexing what is still pending.", + "type": "object", + "properties": { + "Behind": { + "type": "object", + "properties": { + "pending": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "pending" + ] + } + }, + "additionalProperties": false, + "required": [ + "Behind" + ] + }, + { + "description": "The last attempt failed (the embedder or the vector store is\nunreachable, or refused); it is retried with a backoff and nothing is\ndropped. `attempts` counts consecutive failures.", + "type": "object", + "properties": { + "Failing": { + "type": "object", + "properties": { + "attempts": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "error": { + "type": "string" + }, + "pending": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "pending", + "attempts", + "error" + ] + } + }, + "additionalProperties": false, + "required": [ + "Failing" + ] + }, + { + "description": "The collection belongs to another writer. The worker never recreates\nit; it re-reads it on the same backoff.", + "type": "object", + "properties": { + "Refused": { + "type": "object", + "properties": { + "attempts": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "pending": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "reason": { + "type": "string" + } + }, + "required": [ + "pending", + "attempts", + "reason" + ] + } + }, + "additionalProperties": false, + "required": [ + "Refused" + ] + } + ] + }, "Label": { "description": "A key label: `[A-Za-z0-9_-]{1,64}`.", "type": "string", @@ -1314,6 +1436,9 @@ "format": "uint32", "minimum": 0 }, + "index": { + "$ref": "#/$defs/IndexStatus" + }, "instance": { "$ref": "#/$defs/Slug" }, @@ -1330,7 +1455,8 @@ "instance", "schema_epoch", "documents", - "receipts" + "receipts", + "index" ] }, "MutationRecord": { From 38059c959944f23e436bc02d5442cdd2993bd665 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 08:05:27 +0200 Subject: [PATCH 004/520] Cut 11a: a live adapter smoke against a scratch collection (ignored by default) Co-Authored-By: Claude Sonnet 5 --- .../huginn-daemon/tests/live_index_smoke.rs | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 crates/huginn-daemon/tests/live_index_smoke.rs diff --git a/crates/huginn-daemon/tests/live_index_smoke.rs b/crates/huginn-daemon/tests/live_index_smoke.rs new file mode 100644 index 0000000..d1fc545 --- /dev/null +++ b/crates/huginn-daemon/tests/live_index_smoke.rs @@ -0,0 +1,111 @@ +//! A live check of the two adapters against a real Qdrant and a real Ollama. +//! Ignored by default: it needs both services, and it writes to Qdrant, so it +//! touches one scratch collection it creates (`huginn-smoke-`, and a +//! dotted sibling to learn whether Qdrant accepts a `.` in a name) and deletes +//! both at the end. It never names, lists or reads any other collection. +//! +//! HUGINN_SMOKE_QDRANT=http://127.0.0.1:6333 \ +//! HUGINN_SMOKE_OLLAMA=http://10.77.0.4:11434 \ +//! HUGINN_SMOKE_MODEL=qwen3-embedding:0.6b \ +//! cargo test -p huginn-daemon --test live_index_smoke -- --ignored --nocapture + +use std::collections::BTreeSet; + +use huginn_daemon::index::ollama::OllamaEmbedder; +use huginn_daemon::index::qdrant::QdrantIndex; +use huginn_daemon::index::{ + CollectionMeta, Described, Embedder, MANAGED_BY, Point, PointPayload, VectorIndex, point_id, +}; +use huginn_mind::INDEX_TEXT_VERSION; + +fn env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} is required")) +} + +fn raw(method: &str, url: &str, body: Option<&str>) -> (u16, String) { + let agent: ureq::Agent = ureq::Agent::config_builder().http_status_as_error(false).build().into(); + let mut response = match (method, body) { + ("DELETE", _) => agent.delete(url).call(), + ("POST", Some(body)) => agent.post(url).header("content-type", "application/json").send(body), + _ => agent.get(url).call(), + } + .expect("the request was sent"); + let status = response.status().as_u16(); + (status, response.body_mut().read_to_string().unwrap()) +} + +#[test] +#[ignore = "needs a live Qdrant and Ollama, and writes a scratch collection"] +fn the_adapters_embed_write_query_and_clean_up_a_scratch_collection() { + let (qdrant, ollama, model) = (env("HUGINN_SMOKE_QDRANT"), env("HUGINN_SMOKE_OLLAMA"), env("HUGINN_SMOKE_MODEL")); + let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let plain = format!("huginn-smoke-{:x}-{stamp:x}", std::process::id()); + let dotted = format!("{plain}.dotted"); + + let mut embedder = OllamaEmbedder::new(&ollama, &model); + let identity = embedder.model_identity().expect("the model is listed"); + println!("identity: {identity:?}"); + let texts = ["The organ owns the mind.".to_string(), "A recipe for lentil soup.".to_string()]; + let vectors = embedder.embed(&texts).expect("two texts embed"); + assert_eq!(vectors.len(), 2); + assert!(vectors.iter().all(|vector| vector.len() == identity.dimensions as usize)); + + let meta = CollectionMeta { + managed_by: MANAGED_BY.into(), + instance: "smoke".into(), + model: identity.name.clone(), + model_digest: identity.digest.clone(), + dimensions: identity.dimensions, + index_text_version: INDEX_TEXT_VERSION, + }; + let mut store = QdrantIndex::new(&qdrant); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + for name in [&plain, &dotted] { + assert_eq!(store.describe(name).unwrap(), Described::Absent); + store.recreate(name, &meta).unwrap(); + assert_eq!(store.describe(name).unwrap(), Described::Labelled(meta.clone())); + assert!(store.ids(name).unwrap().is_empty()); + let points: Vec = texts + .iter() + .zip(&vectors) + .enumerate() + .map(|(at, (text, vector))| Point { + id: point_id(&format!("smoke:doc:{at}")), + vector: vector.clone(), + payload: PointPayload { + doc_id: format!("smoke:doc:{at}"), + kind: "question".into(), + root: "smoke".into(), + ordinal: at as u64 + 1, + text_sha256: format!("{}", text.len()), + }, + }) + .collect(); + store.upsert(name, &points).unwrap(); + let expected: BTreeSet = points.iter().map(|point| point.id.clone()).collect(); + assert_eq!(store.ids(name).unwrap(), expected); + // The nearest point to the first text is the first point. + let query = serde_json_free_query(&vectors[0]); + let (status, answer) = raw("POST", &format!("{qdrant}/collections/{name}/points/search"), Some(&query)); + assert_eq!(status, 200, "{answer}"); + assert!(answer.contains(&points[0].id), "{answer}"); + println!("{name}: described, created, upserted {} points, queried", points.len()); + } + })); + for name in [&plain, &dotted] { + let (status, _) = raw("DELETE", &format!("{qdrant}/collections/{name}?wait=true"), None); + println!("deleted {name}: {status}"); + } + let (status, _) = raw("GET", &format!("{qdrant}/collections/{plain}"), None); + assert_eq!(status, 404, "the scratch collection is gone"); + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } +} + +/// The nearest-neighbour body, written out by hand so the test needs no JSON +/// crate of its own. +fn serde_json_free_query(vector: &[f32]) -> String { + let numbers: Vec = vector.iter().map(|value| value.to_string()).collect(); + format!("{{\"vector\":[{}],\"limit\":1}}", numbers.join(",")) +} From 18de1bc6d7d42034159871a19a91e7524157fd9d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 08:06:41 +0200 Subject: [PATCH 005/520] BP-3 3: bound every echoed field, and defer under the answer's own status A reply echoes the request's message_id (and on failures its service_id, operation and payload_schema; on chunk answers its chunk_hash). answer now refuses any of them over MAX_ECHOED_FIELD_BYTES (256) at the door with a transport Error that carries none, so every reply, including the largest Deferred and ResponseTooLarge, fits the window. A deferred answer's envelope carries the status of the answer it stands for. The retention docs name the missing per-peer budget and that expiry runs only when a frame arrives. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/bodies.rs | 10 +- crates/huginn-daemon/src/serve.rs | 165 +++++++++++++++++++++++++++-- crates/huginn-mind/src/wire.rs | 3 +- 3 files changed, 165 insertions(+), 13 deletions(-) diff --git a/crates/huginn-daemon/src/bodies.rs b/crates/huginn-daemon/src/bodies.rs index f58d05c..be6de34 100644 --- a/crates/huginn-daemon/src/bodies.rs +++ b/crates/huginn-daemon/src/bodies.rs @@ -11,7 +11,15 @@ //! again and serving any of its chunks both touch it. When the stored bytes //! exceed the budget the body touched longest ago is evicted first. A chunk //! that two bodies share is stored once and lives until the last of them goes. -//! Time is an argument, so nothing here reads a clock. +//! Time is an argument, so nothing here reads a clock; expiry therefore runs +//! only when a frame arrives, and an idle daemon holds an expired body until +//! the next one. +//! +//! Limit: there is no per-session or per-peer budget, so any client can evict +//! another client's body mid-fetch: four 64 MiB reads flush the 256 MiB store, +//! and the loser's next chunk is answered `found: false`. That is acceptable +//! for the organ's trusted WireGuard clients, where an instance identity is +//! declared attribution and not authentication. use std::cell::Cell; use std::collections::{BTreeSet, HashMap}; diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index 08698a3..d7c1d63 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -183,6 +183,40 @@ pub fn schema_registry() -> Result { Ok(registry) } +/// The longest client-supplied string a reply may echo, in bytes. A reply +/// echoes the request's `message_id` (and, on a failure, its `service_id`, +/// `operation` and `payload_schema`; on a chunk answer, its `chunk_hash`), so an +/// unbounded field would make the reply unbounded, and the hub would refuse a +/// send the client then waits on forever. Every echoed field is refused past +/// this at the door, with a transport `Error` that carries none of them. With +/// it, every reply is within the window: the largest, a deferred answer or a +/// refusal, is one 256-byte id, a real operation name, the daemon's own runtime +/// id and, for a deferred answer, a manifest of at most +/// `MAX_DEFERRED_BODY_BYTES / DEFERRED_CHUNK_BYTES` chunk references. Real ids +/// are UUID-sized. +pub const MAX_ECHOED_FIELD_BYTES: usize = 256; + +/// The first field of a request that a reply would echo and that is over +/// `MAX_ECHOED_FIELD_BYTES`. +fn overlong_echo(message: &CultNetMessage) -> Option<&'static str> { + let echoed: Vec<(&'static str, &str)> = match message { + CultNetMessage::OperationRequest { message_id, service_id, operation, payload_schema, .. } => { + vec![ + ("message_id", message_id), + ("service_id", service_id), + ("operation", operation), + ("payload_schema", payload_schema), + ] + } + CultNetMessage::SchemaCatalogRequest { message_id, .. } => vec![("message_id", message_id)], + CultNetMessage::ContentChunkRequest { message_id, chunk_hash, record_key, .. } => { + vec![("message_id", message_id), ("chunk_hash", chunk_hash), ("record_key", record_key)] + } + _ => return None, + }; + echoed.into_iter().find(|(_, value)| value.len() > MAX_ECHOED_FIELD_BYTES).map(|(name, _)| name) +} + /// One message in, one message out. Nothing here reaches a mind except through /// `Daemon::handle`, and an envelope that does not decode never gets that far. /// A chunk request is answered from `bodies` alone, on the session it came in @@ -196,6 +230,13 @@ pub fn answer>( ) -> CultNetMessage { let runtime_id = daemon.runtime_id(); bodies.expire(now); + if let Some(field) = overlong_echo(&message) { + return CultNetMessage::Error { + error: format!("{MIND_SERVICE_ID} refuses a {field} over {MAX_ECHOED_FIELD_BYTES} bytes"), + code: None, + details: None, + }; + } match &message { CultNetMessage::OperationRequest { message_id, operation, .. } => match decode_request(&message) { Ok((message_id, request)) => { @@ -256,8 +297,12 @@ fn encode_or_fail( /// A reply that fits is returned as it is. A larger one whose payload, the /// named MessagePack of the response, is at most `MAX_DEFERRED_BODY_BYTES` is /// packed by CultNet's content plane, retained in `bodies`, and answered with -/// `Deferred` and the manifest. That answer is a small fixed-shape envelope and -/// is never itself measured or deferred. A larger payload is refused by name +/// `Deferred` and the manifest. That answer is bounded, not fixed-shape: the id +/// it echoes is at most `MAX_ECHOED_FIELD_BYTES` and its manifest at most 256 +/// references, so it always fits one send and is never itself measured or +/// deferred. Its envelope status is the deferred answer's own, read from the +/// reply before deferral, so routing on status is the same whole or deferred. A +/// larger payload is refused by name /// with its own size and the deferral bound: nothing is truncated or paginated, /// and the caller narrows its own request. Both answers ride the response /// schema like every other, so a client parses no second vocabulary. @@ -284,8 +329,8 @@ fn deliver( if encoded <= MAX_RESPONSE_BYTES { return reply; } - let payload = match &reply { - CultNetMessage::OperationResponse { payload, .. } => STANDARD.decode(payload), + let (payload, answer_status) = match &reply { + CultNetMessage::OperationResponse { payload, status, .. } => (STANDARD.decode(payload), status.clone()), _ => return not_encodable("only an operation response is deferred".into()), }; let payload = match payload { @@ -311,7 +356,12 @@ fn deliver( ) .expect("the deferral constants are a valid pack"); bodies.retain(&manifest, chunks, now); - encode_or_fail(message_id, operation, &HuginnMindResponse::Deferred(DeferredAnswer { manifest }), runtime_id) + let mut deferred = + encode_or_fail(message_id, operation, &HuginnMindResponse::Deferred(DeferredAnswer { manifest }), runtime_id); + if let CultNetMessage::OperationResponse { status, .. } = &mut deferred { + *status = answer_status; + } + deferred } /// Until `stopping`: expire what timed out, resend what was not acknowledged, @@ -924,9 +974,12 @@ mod tests { fn an_answer_over_the_deferral_bound_is_refused_by_name() { let payload_of = |len: u64| STANDARD.encode(vec![0_u8; len as usize]); let mut bodies = fresh_bodies(); + // The longest id a request may carry: the reply echoes it, so the + // largest deferred answer and the largest refusal are measured at it. + let id = "m".repeat(MAX_ECHOED_FIELD_BYTES); - let at = reply_with_payload("m-b", "view", "huginn-yggdrasil", payload_of(MAX_DEFERRED_BODY_BYTES)); - let deferred = deliver(at, "m-b", "view", "huginn-yggdrasil", &mut bodies, now()); + let at = reply_with_payload(&id, "view", "huginn-yggdrasil", payload_of(MAX_DEFERRED_BODY_BYTES)); + let deferred = deliver(at, &id, "view", "huginn-yggdrasil", &mut bodies, now()); let manifest = deferred_manifest(&deferred); assert_eq!(manifest.size_bytes as u64, MAX_DEFERRED_BODY_BYTES); assert_eq!(manifest.chunks.len() as u64, MAX_DEFERRED_BODY_BYTES / DEFERRED_CHUNK_BYTES as u64); @@ -934,18 +987,18 @@ mod tests { assert_eq!(bodies.body_count(), 1); drop(bodies); - let over = reply_with_payload("m-b", "view", "huginn-yggdrasil", payload_of(MAX_DEFERRED_BODY_BYTES + 1)); + let over = reply_with_payload(&id, "view", "huginn-yggdrasil", payload_of(MAX_DEFERRED_BODY_BYTES + 1)); let mut empty = fresh_bodies(); - let refused = deliver(over, "m-b", "view", "huginn-yggdrasil", &mut empty, now()); + let refused = deliver(over, &id, "view", "huginn-yggdrasil", &mut empty, now()); let (correlation, answered) = decode_response(&refused).unwrap(); - assert_eq!(correlation, "m-b"); + assert_eq!(correlation, id); let HuginnMindResponse::Refused(MindRefusal::ResponseTooLarge { bytes, limit }) = answered.expect("a refusal is an answer, not an envelope failure") else { panic!("one byte over the deferral bound was not refused: {refused:?}"); }; assert_eq!((bytes, limit), (MAX_DEFERRED_BODY_BYTES + 1, MAX_DEFERRED_BODY_BYTES)); - assert!(encoded_len(&refused) <= MAX_FRAGMENT_BYTES as u64, "the refusal is one packet"); + assert!(encoded_len(&refused) <= MAX_RESPONSE_BYTES, "the refusal fits one send at the longest id"); assert_eq!(empty.body_count(), 0, "a refused answer retains nothing"); } @@ -1040,6 +1093,96 @@ mod tests { ); } + /// Every field a reply echoes is bounded at the door, so no reply can + /// outgrow the window: a request whose `message_id`, `service_id`, + /// `operation`, `payload_schema` or `chunk_hash` is over the bound is + /// answered with a transport `Error` that fits and carries none of them, + /// and touches no mind; a field of exactly the bound is served as usual. + /// The 1.3 MB id is the one that made a `Whoami` answer echo past the + /// window, deferred it, and left the client waiting on a send the hub + /// refused. + #[test] + fn an_overlong_echoed_field_is_refused_at_the_door_with_an_answer_that_fits() { + let root = tempfile::tempdir().unwrap(); + let mut daemon = Daemon::open(root.path(), &slug(INSTANCE)).unwrap(); + let registry = schema_registry().unwrap(); + let mut bodies = fresh_bodies(); + let huge = "x".repeat(1_300_000); + let over = "x".repeat(MAX_ECHOED_FIELD_BYTES + 1); + let at = "x".repeat(MAX_ECHOED_FIELD_BYTES); + let whoami = |message_id: &str, service_id: &str, operation: &str, payload_schema: &str| { + let payload = STANDARD.encode(rmp_serde::to_vec_named(&HuginnMindRequest::Whoami).unwrap()); + CultNetMessage::OperationRequest { + message_id: message_id.into(), + service_id: service_id.into(), + operation: operation.into(), + payload_schema: payload_schema.into(), + payload_encoding: "messagepack-base64".into(), + payload, + source_runtime_id: None, + target_runtime_id: None, + } + }; + let chunk = |message_id: &str, chunk_hash: &str, record_key: &str| CultNetMessage::ContentChunkRequest { + message_id: message_id.into(), + chunk_hash: chunk_hash.into(), + record_key: record_key.into(), + expected_size_bytes: 1, + }; + let refused = [ + ("huge message_id", whoami(&huge, MIND_SERVICE_ID, "whoami", MIND_REQUEST_SCHEMA)), + ("message_id", whoami(&over, MIND_SERVICE_ID, "whoami", MIND_REQUEST_SCHEMA)), + ("service_id", whoami("m", &huge, "whoami", MIND_REQUEST_SCHEMA)), + ("operation", whoami("m", MIND_SERVICE_ID, &huge, MIND_REQUEST_SCHEMA)), + ("payload_schema", whoami("m", MIND_SERVICE_ID, "whoami", &huge)), + ( + "catalog message_id", + CultNetMessage::SchemaCatalogRequest { + message_id: huge.clone(), + include_schema_json: Some(true), + schema_ids: None, + kinds: None, + }, + ), + ("chunk message_id", chunk(&huge, "abc", "")), + ("chunk_hash", chunk("m", &huge, "")), + ("record_key", chunk("m", "abc", &huge)), + ]; + for (case, message) in refused { + let reply = answer(&mut daemon, ®istry, &mut bodies, message, now()); + let CultNetMessage::Error { error, .. } = &reply else { + panic!("{case}: expected a transport error, got {reply:?}"); + }; + assert!(error.contains("over 256 bytes"), "{case}: {error}"); + assert!(encoded_len(&reply) <= MAX_FRAGMENT_BYTES as u64, "{case}: the refusal is one packet"); + } + assert_eq!(documents(&mut daemon), 0); + + let reply = answer(&mut daemon, ®istry, &mut bodies, whoami(&at, MIND_SERVICE_ID, "whoami", MIND_REQUEST_SCHEMA), now()); + assert_eq!(decode_response(&reply).unwrap().0, at, "a field of exactly the bound is served"); + let reply = answer(&mut daemon, ®istry, &mut bodies, chunk(&at, &at, &at), now()); + assert!(!found(&reply), "an unknown chunk is answered found: false, not refused at the door"); + } + + /// The envelope's status is the answer's own whether it is sent whole or + /// deferred, so a client that routes on status routes the same way. + #[test] + fn a_deferred_answer_keeps_the_status_of_the_answer_it_stands_for() { + for status in ["rejected", "accepted"] { + let payload = STANDARD.encode(vec![0_u8; MAX_RESPONSE_BYTES as usize]); + let mut reply = reply_with_payload("m-s", "view", "huginn-yggdrasil", payload); + let CultNetMessage::OperationResponse { status: carried, .. } = &mut reply else { unreachable!() }; + *carried = status.into(); + let mut bodies = fresh_bodies(); + let deferred = deliver(reply, "m-s", "view", "huginn-yggdrasil", &mut bodies, now()); + let CultNetMessage::OperationResponse { status: envelope, .. } = &deferred else { + panic!("expected an operation response, got {deferred:?}"); + }; + assert_eq!(envelope, status); + deferred_manifest(&deferred); + } + } + /// One cut spec's own reference, by its cut label. fn spec_ref(daemon: &mut Daemon, cut: &str) -> PipelineRef { let selection = Selection { diff --git a/crates/huginn-mind/src/wire.rs b/crates/huginn-mind/src/wire.rs index 5583016..5f46950 100644 --- a/crates/huginn-mind/src/wire.rs +++ b/crates/huginn-mind/src/wire.rs @@ -125,7 +125,8 @@ impl HuginnMindResponse { /// The envelope's `status`, in the C# reference's vocabulary: `rejected` /// exactly for a refusal, `accepted` otherwise. `Conflict` and /// `AlreadyAdmitted` are answers about the mind, not refusals of the - /// request. + /// request. A `Deferred` is nominally `accepted`; the daemon sets the + /// envelope of a deferred answer to the status of the answer it stands for. pub fn status(&self) -> &'static str { match self { Self::Refused(_) | Self::Admit(PipelineAdmissionOutcome::Refused(_)) => "rejected", From 48518a5df3ec0d5cefc51cca903f7f4d785db9ed Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 08:46:12 +0200 Subject: [PATCH 006/520] Cut 11a: pin the collection name, status arithmetic, worker shutdown and name-only tag entries Mutation survivors from the first cargo-mutants pass; the equivalent ones (point-id bit shaping, a byte-limit boundary) are left. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/index/tests.rs | 55 +++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index f4c17ca..e35f695 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -428,6 +428,13 @@ fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { assert!(body.contains("\"model\":\"m:1\"") && body.contains("\"input\":[\"a\",\"b\",\"c\",\"d\"]"), "{body}"); drop(seen); + // A listing entry that carries only `name` is still the model. + let bare = stub(&[ + ("GET /api/tags", 200, r#"{"models":[{"name":"m:1","digest":"ac6d"}]}"#), + ("POST /api/show", 200, SHOW), + ]); + assert_eq!(OllamaEmbedder::new(&bare.url, "m:1").model_identity().unwrap().digest, "ac6d"); + let mut absent = OllamaEmbedder::new(&ollama.url, "not-pulled:1"); assert!(format!("{:#}", absent.model_identity().unwrap_err()).contains("does not list the model")); let refusing = stub(&[("POST /api/embed", 500, r#"{"error":"boom"}"#)]); @@ -543,3 +550,51 @@ fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { let failing = stub(&[("PUT /collections/", 500, r#"{"status":{"error":"disk full"}}"#)]); assert!(QdrantIndex::new(&failing.url).upsert(&name, &[]).is_err()); } + +/// The collection's name is the contract with the store: one per instance, +/// `huginn_mind_`. +#[test] +fn the_collection_is_named_for_the_instance() { + assert_eq!(collection_name(&slug("yggdrasil")), "huginn_mind_yggdrasil"); + assert_eq!(collection_name(&slug("a.b")), "huginn_mind_a.b"); +} + +#[test] +fn a_step_error_says_what_stopped_it() { + assert_eq!(StepError::Refused("not ours".into()).to_string(), "not ours"); + let unavailable = StepError::Unavailable(anyhow::anyhow!("inner").context("outer")); + assert_eq!(unavailable.to_string(), "outer: inner"); +} + +/// What the sink reports is the worker's health with the inbox added to the +/// pending count, and a current index with something queued is behind. +#[test] +fn queued_entries_count_toward_every_status_that_has_a_pending_count() { + use IndexStatus as S; + assert_eq!(including(&S::Current, 0), S::Current); + assert_eq!(including(&S::Current, 3), S::Behind { pending: 3 }); + assert_eq!(including(&S::Reconciling { pending: 2 }, 3), S::Reconciling { pending: 5 }); + assert_eq!(including(&S::Behind { pending: 2 }, 3), S::Behind { pending: 5 }); + assert_eq!( + including(&S::Failing { pending: 2, attempts: 4, error: "e".into() }, 3), + S::Failing { pending: 5, attempts: 4, error: "e".into() } + ); + assert_eq!( + including(&S::Refused { pending: 2, attempts: 4, reason: "r".into() }, 3), + S::Refused { pending: 5, attempts: 4, reason: "r".into() } + ); +} + +/// Dropping the sink stops the worker: the worker's hold on the shared state +/// is released. A worker that never stopped would hold it forever. +#[test] +fn dropping_the_sink_stops_the_worker() { + let (embedder, index) = pair(); + let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let shared = Arc::downgrade(&sink.shared); + drop(sink); + while shared.upgrade().is_some() { + std::thread::yield_now(); + } +} From 1c1466aa5bf4aa178a1c80ae1d41eb979b5e89a3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:08:37 +0200 Subject: [PATCH 007/520] Mind exposes its identity: the genesis receipt id The instance name is only a label; two stores may share it. The receipt of the first admission never changes and digests the first batch. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-mind/src/admission.rs | 13 +++++++++++++ crates/huginn-mind/src/mind.rs | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/crates/huginn-mind/src/admission.rs b/crates/huginn-mind/src/admission.rs index 8ea8f8a..4b85cdd 100644 --- a/crates/huginn-mind/src/admission.rs +++ b/crates/huginn-mind/src/admission.rs @@ -680,6 +680,19 @@ mod tests { assert!(Mind::open_with(store, &slug(OTHER_INSTANCE)).is_err()); } + /// The identity the index labels its collection with: none before the + /// first write, then the first receipt's id and never a later one. + #[test] + fn the_genesis_receipt_id_is_the_first_admissions_and_stays_put() { + let mut mind = opened(MemoryStore::new(), INSTANCE); + assert_eq!(mind.genesis_receipt_id().unwrap(), None); + let (first, _) = committed(admit(&mut mind, vec![instance(INSTANCE), stewardship(INSTANCE, REPO)])); + assert_eq!(mind.genesis_receipt_id().unwrap(), Some(first.clone())); + let (second, _) = committed(admit(&mut mind, vec![campaign(&[REPO])])); + assert_ne!(first, second); + assert_eq!(mind.genesis_receipt_id().unwrap(), Some(first)); + } + #[test] fn admission_refuses_a_foreign_instance_whatever_the_transport() { let store = MemoryStore::new(); diff --git a/crates/huginn-mind/src/mind.rs b/crates/huginn-mind/src/mind.rs index 8deed10..ff5fbc2 100644 --- a/crates/huginn-mind/src/mind.rs +++ b/crates/huginn-mind/src/mind.rs @@ -245,6 +245,16 @@ impl Mind { .collect() } + /// The mind's own identity: the receipt id of its first admission + /// (ordinal 1), which admission never rewrites and which digests the + /// instance and the first batch's bytes. `None` for a mind that has not + /// been written yet. The instance name is only a label, since two stores + /// on two hosts may share it; this tells them apart unless their first + /// batches were byte-identical. + pub fn genesis_receipt_id(&self) -> Result, MindRefusal> { + Ok(self.receipts()?.into_iter().find(|receipt| receipt.ordinal == 1).map(|receipt| receipt.receipt_id)) + } + pub(crate) fn raw_envelope(&self, type_id: &str, key: &str) -> Option<&CultCacheEnvelope> { self.image.iter().find(|envelope| envelope.r#type == type_id && envelope.key == key) } From 98c959f0f5282b9555d62240eda508388def68e2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:08:38 +0200 Subject: [PATCH 008/520] Index: reconcile after every failure, label collections with the mind, pin the backoff F1 the 30 s to 10 min backoff curve is pinned by a pure test. F2 any failed step forgets the reconciliation; the projector keeps every entry it was given so a lost collection or a changed model is rebuilt and refilled without a restart. F3 the collection label carries the mind genesis receipt id; a collection of another mind with the same instance name is refused and untouched. A mind with no receipt yet reconciles nothing. F4 the effective input limit (model context, not 16 KiB) and the Cut 11b join requirement are recorded in the module docs. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/index/mod.rs | 148 +++++++++++++++---- crates/huginn-daemon/src/index/qdrant.rs | 7 +- crates/huginn-daemon/src/index/tests.rs | 179 +++++++++++++++++++++-- crates/huginn-daemon/src/serve.rs | 2 +- 4 files changed, 290 insertions(+), 46 deletions(-) diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs index e75d7c0..ab864c1 100644 --- a/crates/huginn-daemon/src/index/mod.rs +++ b/crates/huginn-daemon/src/index/mod.rs @@ -18,11 +18,37 @@ //! `Embedder` (`ollama`) and `VectorIndex` (`qdrant`). Those two files are the //! xenos boundary; nothing else in the organ speaks HTTP or JSON. //! -//! Stated limits. The inbox is unbounded: it holds at most the documents -//! admitted while the index is unreachable, each of which is also in the mind. -//! The model identity is read at reconciliation only, so a model re-pulled -//! while the daemon runs takes effect at the next start, and vectors of the -//! wrong length are refused rather than written. +//! Reconciliation is not a startup event. After any failed step the worker +//! reconciles again before it writes anything else: a collection lost while the +//! daemon runs is recreated and refilled, and a model re-pulled with other +//! dimensions rebuilds the collection, all without a restart. Refilling needs +//! the text, so the projector keeps every entry it was given (`known`) beside +//! the ids it still owes (`wanted`): the whole indexable mind's text is held in +//! memory for the process's life. +//! +//! A collection belongs to a mind, not to a name. It is labelled with the +//! instance and with the mind's genesis receipt id (`Mind::genesis_receipt_id`), +//! and a collection labelled for another mind of the same instance name, or +//! for none, is refused and left untouched. A mind with no receipt yet has no +//! identity, so nothing is reconciled until its first admission. +//! +//! Stated limits. +//! +//! - The inbox is unbounded: it holds at most the documents admitted while the +//! index is unreachable, each of which is also in the mind. +//! - Vectors of the wrong length are refused rather than written. +//! - The real input limit is the embedding model's context, not the 16 KiB +//! `INDEX_TEXT_MAX_BYTES` bound. `qwen3-embedding:0.6b` has a 4096 token +//! context, so dense text is cut at about 4k tokens and the tail of a long +//! document is not searchable. Ollama truncates silently by default; its +//! `truncate: false` option refuses over-context input instead, which would +//! fail the whole batch forever on one dense document, so it is not set. +//! There is no chunking. +//! - Points are never deleted. Withdrawn and superseded documents are indexed +//! on purpose and the payload carries no status. Cut 11b's semantic read must +//! therefore join every hit back through the mind: drop ids the mind does not +//! hold, and filter by resolution using RS-3's in-force derivation through +//! selection, never by anything in the payload. pub mod ollama; pub mod qdrant; @@ -63,6 +89,8 @@ pub struct ModelIdentity { pub struct CollectionMeta { pub managed_by: String, pub instance: String, + /// The mind's genesis receipt id: the identity the instance name lacks. + pub mind: String, pub model: String, pub model_digest: String, pub dimensions: u32, @@ -165,28 +193,44 @@ pub struct Projector { embedder: E, index: V, instance: String, + /// `None` until the mind's first admission; nothing is reconciled before. + mind: Option, collection: String, - wanted: BTreeMap, + /// Every entry ever given, by document id: refilling a lost collection + /// needs text the mind's writes have already gone past. + known: BTreeMap, + /// The ids in `known` whose points the collection may lack. + wanted: BTreeSet, dimensions: Option, } impl Projector { - pub fn new(embedder: E, index: V, instance: &Slug) -> Self { + pub fn new(embedder: E, index: V, instance: &Slug, mind: Option) -> Self { Self { embedder, index, instance: instance.0.clone(), + mind, collection: collection_name(instance), - wanted: BTreeMap::new(), + known: BTreeMap::new(), + wanted: BTreeSet::new(), dimensions: None, } } + /// The mind's identity, once it has one. It never changes after that. + fn identify(&mut self, mind: String) { + if self.mind.is_none() { + self.mind = Some(mind); + } + } + /// Entries the collection may lack. Idempotent: an entry already wanted is /// the same document. pub fn want(&mut self, entries: impl IntoIterator) { for entry in entries { - self.wanted.insert(entry.id.id.0.clone(), entry); + self.wanted.insert(entry.id.id.0.clone()); + self.known.insert(entry.id.id.0.clone(), entry); } } @@ -196,7 +240,21 @@ impl Projector { /// One step: reconcile first, then one batch at a time until nothing is /// wanted. + /// + /// Any failure forgets the reconciliation: the next attempt starts from + /// the model and the collection as they are then. pub fn advance(&mut self) -> Result { + let stepped = self.step(); + if stepped.is_err() { + self.dimensions = None; + } + stepped + } + + fn step(&mut self) -> Result { + if self.mind.is_none() { + return Ok(Advance::Idle); + } if self.dimensions.is_none() { self.reconcile()?; return Ok(Advance::Progressed); @@ -211,13 +269,17 @@ impl Projector { /// The only place a point is judged missing. Reads the model identity, /// makes the collection agree with it (creating it, or rebuilding it when /// the model, its digest, its dimensions or the text version differ), - /// refuses a collection that is not this instance's, then drops from - /// `wanted` every entry whose point the collection already holds. + /// refuses a collection that is not this mind's, then makes `wanted` every + /// known entry whose point the collection does not hold. pub fn reconcile(&mut self) -> Result<(), StepError> { + let Some(mind) = self.mind.clone() else { + return Err(StepError::Unavailable(anyhow::anyhow!("the mind has no identity yet"))); + }; let identity = self.embedder.model_identity().context("reading the embedding model's identity")?; let wanted = CollectionMeta { managed_by: MANAGED_BY.into(), instance: self.instance.clone(), + mind: mind.clone(), model: identity.name.clone(), model_digest: identity.digest.clone(), dimensions: identity.dimensions, @@ -231,17 +293,19 @@ impl Projector { self.collection ))); } - Described::Labelled(found) if found.managed_by != MANAGED_BY || found.instance != self.instance => { + Described::Labelled(found) + if found.managed_by != MANAGED_BY || found.instance != self.instance || found.mind != mind => + { return Err(StepError::Refused(format!( - "collection {} is managed by {:?} for instance {:?}; this organ serves {:?}", - self.collection, found.managed_by, found.instance, self.instance + "collection {} is managed by {:?} for instance {:?}, mind {:?}; this organ serves instance {:?}, mind {:?}", + self.collection, found.managed_by, found.instance, found.mind, self.instance, mind ))); } Described::Labelled(found) if found != wanted => self.recreate(&wanted)?, Described::Labelled(_) => {} } let present = self.index.ids(&self.collection).context("listing the collection's points")?; - self.wanted.retain(|document_id, _| !present.contains(&point_id(document_id))); + self.wanted = self.known.keys().filter(|document_id| !present.contains(&point_id(document_id))).cloned().collect(); self.dimensions = Some(identity.dimensions); Ok(()) } @@ -259,7 +323,8 @@ impl Projector { fn flush(&mut self) -> Result<()> { let dimensions = self.dimensions.context("flush before reconcile")? as usize; - let batch: Vec = self.wanted.values().take(BATCH).cloned().collect(); + let batch: Vec = + self.wanted.iter().take(BATCH).filter_map(|document_id| self.known.get(document_id)).cloned().collect(); if batch.is_empty() { return Ok(()); } @@ -296,6 +361,7 @@ impl Projector { /// The health after a step that did not fail. pub fn progress_status(&self) -> IndexStatus { match (self.dimensions, self.pending()) { + (None, 0) if self.mind.is_none() => IndexStatus::Current, (None, pending) => IndexStatus::Reconciling { pending }, (Some(_), 0) => IndexStatus::Current, (Some(_), pending) => IndexStatus::Behind { pending }, @@ -335,6 +401,8 @@ impl Backoff { struct Shared { inbox: Vec, + /// The mind's identity, once a commit has revealed it to a mind that had none. + mind: Option, status: IndexStatus, closed: bool, } @@ -345,14 +413,14 @@ fn lock(shared: &Lock) -> MutexGuard<'_, Shared> { shared.0.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) } -/// With no timeout, blocks until the inbox holds something or the sink is -/// closed. With one, sleeps out the backoff whatever arrives: a failing index +/// With no timeout, blocks until the inbox holds something, the mind reveals +/// its identity, or the sink is closed. With one, sleeps out the backoff whatever arrives: a failing index /// is retried on its schedule, not once per admission. fn wait_for_work(shared: &Lock, timeout: Option) { let mut guard = lock(shared); match timeout { None => { - while guard.inbox.is_empty() && !guard.closed { + while guard.inbox.is_empty() && guard.mind.is_none() && !guard.closed { guard = shared.1.wait(guard).unwrap_or_else(|poisoned| poisoned.into_inner()); } } @@ -396,13 +464,16 @@ fn work(mut projector: Projector, shared: Loc let mut failures = 0_u32; let mut delay = backoff.initial; loop { - let inbox = { + let (mind, inbox) = { let mut guard = lock(&shared); if guard.closed { return; } - std::mem::take(&mut guard.inbox) + (guard.mind.take(), std::mem::take(&mut guard.inbox)) }; + if let Some(mind) = mind { + projector.identify(mind); + } projector.want(inbox); // While a failure is being reported, the report stands until an // attempt succeeds: publishing progress between retries would flicker. @@ -442,25 +513,37 @@ fn work(mut projector: Projector, shared: Loc /// it: a call in flight ends with its own timeout or with the process. pub struct WorkerSink { shared: Lock, + /// Whether the worker has been told the mind's identity. + identified: bool, } impl WorkerSink { - /// Starts the worker. `startup` is every indexable document the mind holds - /// now; the worker's first act is to reconcile the collection against it. - pub fn spawn(embedder: E, index: V, instance: &Slug, startup: Vec, backoff: Backoff) -> Self + /// Starts the worker. `mind` is the mind's genesis receipt id, `None` for + /// a mind not yet written. `startup` is every indexable document the mind + /// holds now; the worker's first act is to reconcile the collection + /// against it. + pub fn spawn( + embedder: E, + index: V, + instance: &Slug, + mind: Option, + startup: Vec, + backoff: Backoff, + ) -> Self where E: Embedder + Send + 'static, V: VectorIndex + Send + 'static, { - let mut projector = Projector::new(embedder, index, instance); + let identified = mind.is_some(); + let mut projector = Projector::new(embedder, index, instance, mind); projector.want(startup); let shared: Lock = Arc::new(( - Mutex::new(Shared { inbox: Vec::new(), status: projector.progress_status(), closed: false }), + Mutex::new(Shared { inbox: Vec::new(), mind: None, status: projector.progress_status(), closed: false }), Condvar::new(), )); let worker = Arc::clone(&shared); std::thread::spawn(move || work(projector, worker, backoff)); - Self { shared } + Self { shared, identified } } /// Blocks until the health satisfies `predicate`, and returns it. For @@ -494,10 +577,17 @@ impl Drop for WorkerSink { impl IndexSink for WorkerSink { fn committed(&mut self, mind: &Mind, writes: &[PipelineRef]) -> Result<()> { let entries = mind.index_entries(Some(writes))?; - if entries.is_empty() { + let revealed = if self.identified { None } else { mind.genesis_receipt_id()? }; + if entries.is_empty() && revealed.is_none() { return Ok(()); } - lock(&self.shared).inbox.extend(entries); + let mut guard = lock(&self.shared); + guard.inbox.extend(entries); + if let Some(revealed) = revealed { + guard.mind = Some(revealed); + self.identified = true; + } + drop(guard); self.shared.1.notify_all(); Ok(()) } diff --git a/crates/huginn-daemon/src/index/qdrant.rs b/crates/huginn-daemon/src/index/qdrant.rs index c119f7e..a794ac1 100644 --- a/crates/huginn-daemon/src/index/qdrant.rs +++ b/crates/huginn-daemon/src/index/qdrant.rs @@ -36,6 +36,7 @@ fn metadata_json(meta: &CollectionMeta) -> Value { json!({ "managed_by": meta.managed_by, "instance": meta.instance, + "mind": meta.mind, "model": meta.model, "model_digest": meta.model_digest, "dimensions": meta.dimensions, @@ -45,8 +46,9 @@ fn metadata_json(meta: &CollectionMeta) -> Value { /// What a collection's `config.metadata` says. A collection whose metadata /// does not name a writer and an instance is unlabelled; one that does but is -/// missing the rest reads as an empty model, which will not match and will be -/// rebuilt if the writer and the instance are ours. +/// missing the rest reads as an empty mind or model, which will not match: an +/// empty mind is refused as another's, an empty model is rebuilt when the +/// writer, the instance and the mind are ours. fn described(metadata: &Value) -> Described { let text = |key: &str| metadata[key].as_str().map(str::to_owned); let (Some(managed_by), Some(instance)) = (text("managed_by"), text("instance")) else { @@ -56,6 +58,7 @@ fn described(metadata: &Value) -> Described { Described::Labelled(CollectionMeta { managed_by, instance, + mind: text("mind").unwrap_or_default(), model: text("model").unwrap_or_default(), model_digest: text("model_digest").unwrap_or_default(), dimensions: number("dimensions"), diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index e35f695..60ef39b 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -23,6 +23,22 @@ fn collection() -> String { collection_name(&slug(INSTANCE)) } +/// A mind identity for tests that project entries without a mind behind them. +const MIND: &str = "mind-commit-first"; + +/// A mind of the same instance name whose first batch differs, so its genesis +/// receipt id does too. +fn other_mind_of_the_same_name() -> (TempDir, Mind) { + let root = tempfile::tempdir().unwrap(); + let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (question, _) = question_and_ruling(); + let mut first = campaign_seed(); + first.push(question); + let outcome = mind.admit(batch(INSTANCE, first), now()); + assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); + (root, mind) +} + /// A real mind holding a campaign, a question, the ruling that answers it and /// the resolution admission derived: four indexable documents, beside the /// identity and the stewardship, which carry no text. @@ -52,7 +68,7 @@ fn point_ids(entries: &[IndexEntry]) -> BTreeSet { } fn projected(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE)); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); projector.want(entries.iter().cloned()); drain(&mut projector); projector @@ -166,9 +182,10 @@ fn a_model_digest_change_rebuilds_the_collection() { fn a_collection_owned_by_another_instance_is_never_recreated() { let (_root, mind) = mind_with_documents(); let entries = mind.index_entries(None).unwrap(); - let ours = |managed_by: &str, instance: &str| CollectionMeta { + let ours = |managed_by: &str, instance: &str, mind: &str| CollectionMeta { managed_by: managed_by.into(), instance: instance.into(), + mind: mind.into(), model: "model".into(), model_digest: "d1".into(), dimensions: 4, @@ -176,8 +193,10 @@ fn a_collection_owned_by_another_instance_is_never_recreated() { }; for foreign in [ Described::Unlabelled, - Described::Labelled(ours(MANAGED_BY, OTHER)), - Described::Labelled(ours("voidbot", INSTANCE)), + Described::Labelled(ours(MANAGED_BY, OTHER, MIND)), + Described::Labelled(ours("voidbot", INSTANCE, MIND)), + Described::Labelled(ours(MANAGED_BY, INSTANCE, "mind-commit-another")), + Described::Labelled(ours(MANAGED_BY, INSTANCE, "")), ] { let (embedder, index) = pair(); index.plant(&collection(), foreign.clone()); @@ -196,7 +215,7 @@ fn a_collection_owned_by_another_instance_is_never_recreated() { }, }, ); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE)); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); projector.want(entries.iter().cloned()); let error = projector.reconcile().expect_err("a foreign collection is refused"); assert!(matches!(error, StepError::Refused(_)), "{foreign:?}: {error}"); @@ -216,7 +235,7 @@ fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { let (_root, mind) = mind_with_documents(); let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE)); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); projector.want(entries.iter().cloned()); projector.reconcile().unwrap(); embedder.state.lock().unwrap().identity.dimensions = 8; @@ -255,7 +274,7 @@ fn an_admission_replies_while_the_embedder_is_blocked() { let gate = Gate::shut(); let embedder = FakeEmbedder::new("d1").behind(&gate); let index = FakeIndex::default(); - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.index_entries(None).unwrap(), quick()); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), mind.index_entries(None).unwrap(), quick()); let mut daemon = Daemon::new(mind, sink); daemon.index().wait_status(|status| *status == IndexStatus::Current); @@ -283,7 +302,7 @@ fn a_failing_embedder_is_visible_in_whoami_and_retried_never_dropped() { let root = tempfile::tempdir().unwrap(); let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), Vec::new(), quick()); + let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); let mut daemon = Daemon::new(mind, sink); daemon.index().wait_status(|status| *status == IndexStatus::Current); @@ -313,7 +332,7 @@ fn a_refused_collection_is_reported_and_reread() { let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); index.plant(&collection(), Described::Unlabelled); - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), entries, quick()); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries, quick()); let mut daemon = Daemon::new(mind, sink); daemon.index().wait_status(|status| matches!(status, IndexStatus::Refused { attempts, .. } if *attempts >= 2)); assert!(matches!(whoami_index(&mut daemon), IndexStatus::Refused { pending: 4, .. })); @@ -452,13 +471,13 @@ fn the_embedders_address_is_not_part_of_the_collections_compatibility() { let repulled = TAGS.replace("ac6d", "beef"); let index = FakeIndex::default(); - let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE)); + let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); first.want(entries.iter().cloned()); drain(&mut first); assert_eq!(counts(&index), (1, 1)); // Another address, the same model and digest. - let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE)); + let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); moved.want(entries.iter().cloned()); drain(&mut moved); assert_eq!(counts(&index), (1, 1)); @@ -466,7 +485,7 @@ fn the_embedders_address_is_not_part_of_the_collections_compatibility() { // The same model name re-pulled: another digest. let leaked: &'static str = Box::leak(repulled.into_boxed_str()); let other = [("GET /api/tags", 200, leaked), routes[1], routes[2]]; - let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE)); + let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); rebuilt.want(entries.iter().cloned()); drain(&mut rebuilt); assert_eq!(counts(&index), (2, 2)); @@ -486,13 +505,14 @@ fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { let labelled = stub(&[( "GET /collections/", 200, - r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, + r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","mind":"mind-commit-first","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, )]); assert_eq!( QdrantIndex::new(&labelled.url).describe(&name).unwrap(), Described::Labelled(CollectionMeta { managed_by: "huginn".into(), instance: "yggdrasil".into(), + mind: "mind-commit-first".into(), model: "m:1".into(), model_digest: "ac6d".into(), dimensions: 4, @@ -517,6 +537,7 @@ fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { let meta = CollectionMeta { managed_by: MANAGED_BY.into(), instance: INSTANCE.into(), + mind: MIND.into(), model: "m:1".into(), model_digest: "ac6d".into(), dimensions: 4, @@ -590,7 +611,7 @@ fn queued_entries_count_toward_every_status_that_has_a_pending_count() { #[test] fn dropping_the_sink_stops_the_worker() { let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Vec::new(), quick()); + let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); sink.wait_status(|status| *status == IndexStatus::Current); let shared = Arc::downgrade(&sink.shared); drop(sink); @@ -598,3 +619,133 @@ fn dropping_the_sink_stops_the_worker() { std::thread::yield_now(); } } + +/// F1. The promised curve, pure: 30 s doubling to a 10 min cap, never below +/// the first delay and never past the cap. A retry that shrank would hot-loop +/// against a dead Ollama. +#[test] +fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() { + let backoff = Backoff::default(); + assert_eq!((backoff.initial, backoff.max), (Duration::from_secs(30), Duration::from_secs(600))); + let mut delay = backoff.initial; + let mut seen = vec![delay.as_secs()]; + for _ in 0..7 { + let next = backoff.after(delay); + assert!(next >= delay && next >= backoff.initial, "the delay never shrinks: {delay:?} -> {next:?}"); + assert!(next <= backoff.max, "the delay never passes the cap: {next:?}"); + delay = next; + seen.push(delay.as_secs()); + } + assert_eq!(seen, [30, 60, 120, 240, 480, 600, 600, 600]); + let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3) }; + assert_eq!(small.after(Duration::from_millis(1)), Duration::from_millis(2)); + assert_eq!(small.after(Duration::from_millis(2)), Duration::from_millis(3)); +} + +/// F2. A collection lost while the daemon runs is found by the next failed +/// write, recreated and refilled from what the projector knows, without a +/// restart. +#[test] +fn a_collection_lost_after_startup_is_recreated_and_refilled() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 3); + + index.state.lock().unwrap().collections.remove(&collection()); + projector.want(entries[3..].iter().cloned()); + let error = projector.advance().expect_err("the write meets no collection"); + assert!(matches!(error, StepError::Unavailable(_)), "{error}"); + assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Failing { pending: 1, .. })); + + drain(&mut projector); + assert_eq!(index.state.lock().unwrap().recreated.len(), 2, "the collection was made again"); + assert_eq!(index.holding(&collection()), point_ids(&entries), "and holds every document, not only the new one"); + assert_eq!(projector.progress_status(), IndexStatus::Current); +} + +/// F2. A model re-pulled with other dimensions fails one write, then the next +/// attempt reads the new identity and rebuilds the collection to match. +#[test] +fn a_model_that_changes_dimensions_after_startup_rebuilds_the_collection() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + + embedder.state.lock().unwrap().identity.dimensions = 8; + projector.want(entries[3..].iter().cloned()); + assert!(projector.advance().is_err(), "a vector of 8 does not fit a collection of 4"); + drain(&mut projector); + + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2); + assert_eq!(state.recreated.last().unwrap().1.dimensions, 8); + let stored = &state.collections[&collection()]; + assert_eq!(stored.points.len(), 4); + assert!(stored.points.values().all(|point| point.vector.len() == 8)); +} + +/// F3. Two minds that share an instance name never share a collection: the +/// second is refused, and the first's points, label and history are untouched. +#[test] +fn a_second_mind_of_the_same_name_is_refused_and_leaves_the_first_alone() { + let (_first_root, first) = mind_with_documents(); + let (_second_root, second) = other_mind_of_the_same_name(); + let (first_id, second_id) = (first.genesis_receipt_id().unwrap().unwrap(), second.genesis_receipt_id().unwrap().unwrap()); + assert_ne!(first_id, second_id); + + let (embedder, index) = pair(); + let mut ours = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(first_id.clone())); + ours.want(first.index_entries(None).unwrap()); + drain(&mut ours); + let held = index.holding(&collection()); + assert_eq!(held.len(), 4); + + let intruder = FakeEmbedder::new("d1"); + let mut theirs = Projector::new(intruder.clone(), index.clone(), &slug(INSTANCE), Some(second_id)); + theirs.want(second.index_entries(None).unwrap()); + let error = theirs.advance().expect_err("another mind's collection is refused"); + assert!(matches!(error, StepError::Refused(_)), "{error}"); + assert!(matches!(theirs.failure_status(&error, 1), IndexStatus::Refused { .. })); + + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 1, "never rebuilt by the second mind"); + assert_eq!(state.upserts.len(), 1, "never written by the second mind"); + assert!(intruder.texts().is_empty()); + let Described::Labelled(label) = &state.collections[&collection()].label else { panic!() }; + assert_eq!(label.mind, first_id); + drop(state); + assert_eq!(index.holding(&collection()), held); +} + +/// A mind with no receipt has no identity: nothing is reconciled and no +/// collection is made, until the first admission reveals one. +#[test] +fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + assert_eq!(mind.genesis_receipt_id().unwrap(), None); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert_eq!(projector.advance().unwrap(), Advance::Idle); + assert!(index.state.lock().unwrap().collections.is_empty()); + + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); + let mut daemon = Daemon::new(mind, sink); + let seed = campaign_seed(); + let without_text = vec![seed[0].clone(), seed[1].clone()]; + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, without_text)), now())); + for _ in 0..500 { + if index.state.lock().unwrap().collections.contains_key(&collection()) { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let state = index.state.lock().unwrap(); + let Described::Labelled(label) = &state.collections[&collection()].label else { panic!("no collection was made") }; + assert_eq!(Some(&label.mind), daemon.mind().genesis_receipt_id().unwrap().as_ref()); +} diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index 9a1b91d..287f7b9 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -171,7 +171,7 @@ where let mind = Mind::open(&options.state_root, &options.instance)?; let hub = bind(options.bind, &runtime_id(&options.instance))?; let startup_entries = mind.index_entries(None)?; - let sink = WorkerSink::spawn(embedder, index, &options.instance, startup_entries, backoff); + let sink = WorkerSink::spawn(embedder, index, &options.instance, mind.genesis_receipt_id()?, startup_entries, backoff); Ok((Daemon::new(mind, sink), hub, schema_registry()?)) } From 108ed8168c7bbcb51d0650d76b78f56a72909906 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:09:43 +0200 Subject: [PATCH 009/520] Live index smoke labels its scratch collections with a mind Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/tests/live_index_smoke.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/huginn-daemon/tests/live_index_smoke.rs b/crates/huginn-daemon/tests/live_index_smoke.rs index d1fc545..8baaf69 100644 --- a/crates/huginn-daemon/tests/live_index_smoke.rs +++ b/crates/huginn-daemon/tests/live_index_smoke.rs @@ -53,6 +53,7 @@ fn the_adapters_embed_write_query_and_clean_up_a_scratch_collection() { let meta = CollectionMeta { managed_by: MANAGED_BY.into(), instance: "smoke".into(), + mind: "mind-commit-smoke".into(), model: identity.name.clone(), model_digest: identity.digest.clone(), dimensions: identity.dimensions, From 745803c785a21cf051268cb8a749fe2a51dbd4ab Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:26:15 +0200 Subject: [PATCH 010/520] Pin the identified-but-unreconciled status the mutant run found unpinned Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/index/tests.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index 60ef39b..90524b8 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -732,6 +732,8 @@ fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); assert_eq!(projector.progress_status(), IndexStatus::Current); assert_eq!(projector.advance().unwrap(), Advance::Idle); + let identified = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + assert_eq!(identified.progress_status(), IndexStatus::Reconciling { pending: 0 }, "an identified mind has yet to reconcile"); assert!(index.state.lock().unwrap().collections.is_empty()); let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); From 5eea49f756fef12531df6bf512f154e89ecb5159 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 13:37:01 +0200 Subject: [PATCH 011/520] Cut 11b: the semantic read, joined back through the mind Qdrant supplies candidate ids and scores and nothing else. Mind::rank drops every hit the mind does not hold, then lets CultNet's selection decide membership over the whole mind with its keys narrowed to the candidates, so in_force and cited mean what they mean in query. Order is score then ordinal; the page is cut to the selection's limit and top_k; next is never set. A cursor with semantic, blank text and an out-of-range top_k are refused by name before anything is embedded. Mind::query loses its semantic parameter and the "index is not wired" refusal; the published request schema's description of SemanticQuery follows. The serve loop never waits on the embedder: a semantic query becomes a Search held beside its session, the worker embeds (behind the model's query instruction) and searches between batches, and the loop answers when the worker has or when ServeOptions::search_timeout passes. A worker that is Failing or Refused refuses at once, typed Unavailable naming its own error. VectorIndex gains search; Qdrant's adapter sends no filter. Model identity is re-read before every flush and before every search: a model re-pulled under the same dimensions but a new digest forgets the reconciliation instead of mixing vectors, and the next step rebuilds. An idle worker forgets the reconciliation every Backoff::recheck (60 s), so a lost collection is found without a write. Both go through Projector::reconcile, still the one judge. Also fixes a race in the worker loop found by the mutation runs: entries were taken from the inbox under one lock and published as wanted under another, so whoami could read a stale Current with nothing queued while the worker held them. Taking, absorbing and publishing are now one critical section. Co-Authored-By: Claude Sonnet 5 --- README.md | 8 +- crates/huginn-daemon/src/daemon.rs | 200 ++++++-- crates/huginn-daemon/src/index/fakes.rs | 26 +- crates/huginn-daemon/src/index/mod.rs | 279 ++++++++--- crates/huginn-daemon/src/index/qdrant.rs | 18 +- crates/huginn-daemon/src/index/tests.rs | 255 +++++++++- crates/huginn-daemon/src/lib.rs | 5 +- crates/huginn-daemon/src/semantic_tests.rs | 434 ++++++++++++++++++ crates/huginn-daemon/src/serve.rs | 168 +++++-- .../huginn-daemon/tests/live_index_smoke.rs | 14 +- crates/huginn-mind/src/query.rs | 419 +++++++++++++---- .../huginn.mind_request.v1.schema.json | 2 +- 12 files changed, 1573 insertions(+), 255 deletions(-) create mode 100644 crates/huginn-daemon/src/semantic_tests.rs diff --git a/README.md b/README.md index 695c5ff..0b7a07f 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,12 @@ Upstream: `https://github.com/GameCult/Huginn.git` restart compares the mind with the collection and writes what is missing. An unreachable Qdrant or Ollama shows in `whoami` as the index's status and is retried; admission never waits on it. The daemon requires `--qdrant-url`, - `--ollama-url` and `--embedding-model`. The semantic read side is not wired - yet. + `--ollama-url` and `--embedding-model`. A `query` carrying `semantic` is + answered through the same selection as any other: the index supplies + candidate ids and scores, and the mind keeps only the documents it holds and + the selection admits (ask for `in_force` to exclude resolved ones), ordered by + score. It takes no cursor, and it is refused `Unavailable` while the index is + failing or refused. - Mind state is not version-controlled. It lives in Huginn's store, not in any repository. diff --git a/crates/huginn-daemon/src/daemon.rs b/crates/huginn-daemon/src/daemon.rs index 5742e61..ceb86ef 100644 --- a/crates/huginn-daemon/src/daemon.rs +++ b/crates/huginn-daemon/src/daemon.rs @@ -3,18 +3,67 @@ //! thing that reads a wall clock. use chrono::{DateTime, Utc}; +use cultnet_rs::Selection; use huginn_mind::epiphany_pipeline::{PipelineRef, Slug}; use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, IndexStatus}; -use huginn_mind::{Mind, MindStore, PipelineAdmissionOutcome}; +use huginn_mind::{Mind, MindRefusal, MindStore, PipelineAdmissionOutcome, SemanticQuery}; /// Where the index attaches. Called after a batch commits, with the refs the /// batch landed, derived writes included; the sink may read each through /// `mind.view`. Its error is logged and never reaches the caller: the index /// does not decide whether a batch was admitted, and the call must not wait on /// the index's transport. `status` is what `whoami` reports of it. +/// +/// A semantic query is asked in two steps because the index answers on its own +/// time: `search` starts one and returns a ticket at once, and `searched` +/// hands over the searches that have finished since it was last called. The +/// candidates are the index's word about nearness and nothing else; the daemon +/// judges them through the mind. pub trait IndexSink { fn committed(&mut self, mind: &Mind, writes: &[PipelineRef]) -> anyhow::Result<()>; fn status(&self) -> IndexStatus; + /// Starts a search for the documents nearest `text`, at most `top_k` of + /// them wanted. `Err` says why the index cannot be asked now. + fn search(&mut self, text: &str, top_k: u32) -> Result; + /// Every search that finished since the last call, by ticket: its + /// candidates and scores, or why it failed. + fn searched(&mut self) -> Vec<(SearchTicket, Result)>; +} + +/// Names one search the index was asked, so the loop can match the answer to +/// the client that is waiting for it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SearchTicket(pub u64); + +/// What a search returns: candidate documents and how near each scored, best +/// first. Nothing else about them. +pub type Hits = Vec<(PipelineRef, f32)>; + +/// A semantic query the index has been asked and has not yet answered. The +/// serve loop holds it beside the session that asked, and hands it back to +/// `Daemon::finish` with the index's answer. +pub struct Search { + pub ticket: SearchTicket, + selection: Selection, + semantic: SemanticQuery, +} + +/// What a request came to: an answer, or a search the index has yet to +/// finish. +pub enum Handled { + Answered(HuginnMindResponse), + Searching(Search), +} + +impl Handled { + /// The answer, for a request that cannot be a search. + #[cfg(test)] + pub(crate) fn answered(self) -> HuginnMindResponse { + match self { + Self::Answered(response) => response, + Self::Searching(search) => panic!("expected an answer, got the search {:?}", search.ticket), + } + } } /// The envelope's `source_runtime_id` on every response a daemon serving this @@ -54,20 +103,20 @@ impl> Daemon { runtime_id(self.mind.instance()) } - /// One request, one answer. `Admit` goes to the mind whole, because the + /// One request, one answer, or a search to finish. `Admit` goes to the mind whole, because the /// batch declares its own instance and admission's A1 is the check. Every /// other request names an instance: `require_instance` now applies the /// leaf's own grammar before it compares identity, so a name outside the /// grammar (a fullwidth fold, say) is refused by name rather than read as /// merely a foreign mind, and the daemon asks nothing twice. - pub fn handle(&mut self, request: HuginnMindRequest, now: DateTime) -> HuginnMindResponse { + pub fn handle(&mut self, request: HuginnMindRequest, now: DateTime) -> Handled { if let Some(declared) = request.instance() && !matches!(request, HuginnMindRequest::Admit(_)) && let Err(refusal) = self.mind.require_instance(declared) { - return HuginnMindResponse::Refused(refusal); + return Handled::Answered(HuginnMindResponse::Refused(refusal)); } - match request { + let answer = match request { HuginnMindRequest::Whoami => HuginnMindResponse::Whoami(self.mind.status(self.index.status())), HuginnMindRequest::Admit(batch) => { let outcome = self.mind.admit(batch, now); @@ -83,10 +132,57 @@ impl> Daemon { Ok(view) => HuginnMindResponse::View(view), Err(refusal) => HuginnMindResponse::Refused(refusal), }, - HuginnMindRequest::Query { selection, semantic, .. } => match self.mind.query(&selection, semantic.as_ref()) { + HuginnMindRequest::Query { selection, semantic: None, .. } => match self.mind.query(&selection) { Ok(page) => HuginnMindResponse::Query(page), Err(refusal) => HuginnMindResponse::Refused(refusal), }, + HuginnMindRequest::Query { selection, semantic: Some(semantic), .. } => { + return self.begin_search(selection, semantic); + } + }; + Handled::Answered(answer) + } + + /// A semantic query starts only if the mind would accept it (so nothing is + /// embedded for a query that cannot be ranked) and the index can be asked. + fn begin_search(&mut self, selection: Selection, semantic: SemanticQuery) -> Handled { + let refused = |refusal: MindRefusal| Handled::Answered(HuginnMindResponse::Refused(refusal)); + if let Err(refusal) = self.mind.check_semantic(&selection, &semantic) { + return refused(refusal); + } + match self.index.search(&semantic.text.0, semantic.top_k) { + Ok(ticket) => Handled::Searching(Search { ticket, selection, semantic }), + Err(detail) => refused(MindRefusal::Unavailable { detail }), + } + } + + /// Searches the index has finished since this was last called. + pub fn searched(&mut self) -> Vec<(SearchTicket, Result)> { + self.index.searched() + } + + /// The ids among `hits` that the mind holds no document for: the index + /// keeps points the mind no longer has (a restored mind, a second store), + /// and a hit is never shown on the index's word alone. + pub(crate) fn unheld<'a>(&self, hits: &'a Hits) -> Vec<&'a str> { + hits.iter().filter(|(id, _)| !matches!(self.mind.view(id), Ok(Some(_)))).map(|(id, _)| id.id.0.as_str()).collect() + } + + /// The answer to a search: the index's candidates ranked through the mind + /// (`Mind::rank` drops what the mind does not hold and lets the selection + /// decide the rest), or, when the index could not answer, a typed + /// `Unavailable`. Never an empty page in place of a failure. + pub fn finish(&mut self, search: Search, found: Result) -> HuginnMindResponse { + let hits = match found { + Ok(hits) => hits, + Err(detail) => return HuginnMindResponse::Refused(MindRefusal::Unavailable { detail }), + }; + for id in self.unheld(&hits) { + eprintln!("huginn: the index returned {id}, which this mind does not hold; it is left out"); + } + match self.mind.rank(&search.selection, &search.semantic, &hits) { + Ok(page) => HuginnMindResponse::Query(page), + Err(refusal) => HuginnMindResponse::Refused(refusal), } } } @@ -105,7 +201,7 @@ pub(crate) mod tests { use huginn_mind::{MindRefusal, OwnedRedbMessagePackBackingStore}; use std::path::Path; use cultnet_rs::Selection; - use huginn_mind::{Faculty, PipelineAdmissionBatch, PipelineProvenance, PipelinePageItems, PipelineStatus, SemanticQuery}; + use huginn_mind::{Faculty, PipelineAdmissionBatch, PipelineProvenance, PipelinePageItems, PipelineStatus}; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -300,7 +396,7 @@ pub(crate) mod tests { campaign_seed(), vec![cut_spec(WIDE_CUT, WIDE_CHANGES), cut_spec(FITTING_CUT, FITTING_CHANGES)], ] { - let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), now()); + let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), now()).answered(); assert!( matches!(outcome, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), "{outcome:?}" @@ -315,7 +411,7 @@ pub(crate) mod tests { pub(crate) fn seeded() -> (TempDir, Daemon) { let root = tempfile::tempdir().unwrap(); let mut daemon = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); - let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); + let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()).answered(); assert!( matches!(outcome, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), "{outcome:?}" @@ -334,6 +430,14 @@ pub(crate) mod tests { fn status(&self) -> IndexStatus { IndexStatus::Current } + + fn search(&mut self, _text: &str, _top_k: u32) -> Result { + Err("this index cannot search".into()) + } + + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { + Vec::new() + } } /// A daemon over the instance's mind under `state_root`, with no index. @@ -360,6 +464,14 @@ pub(crate) mod tests { fn status(&self) -> IndexStatus { IndexStatus::Current } + + fn search(&mut self, _text: &str, _top_k: u32) -> Result { + Err("this index cannot search".into()) + } + + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { + Vec::new() + } } struct FailingIndex; @@ -372,10 +484,18 @@ pub(crate) mod tests { fn status(&self) -> IndexStatus { IndexStatus::Current } + + fn search(&mut self, _text: &str, _top_k: u32) -> Result { + Err("this index cannot search".into()) + } + + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { + Vec::new() + } } fn status(daemon: &mut Daemon>) -> MindStatus { - match daemon.handle(HuginnMindRequest::Whoami, now()) { + match daemon.handle(HuginnMindRequest::Whoami, now()).answered() { HuginnMindResponse::Whoami(status) => status, other => panic!("expected a status, got {other:?}"), } @@ -387,7 +507,7 @@ pub(crate) mod tests { fn a_batch_round_trips_typed_through_handle_without_a_socket() { let root = tempfile::tempdir().unwrap(); let mut daemon = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); - let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); + let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()).answered(); let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = outcome else { panic!("expected a commit"); }; @@ -410,7 +530,7 @@ pub(crate) mod tests { let HuginnMindResponse::Query(page) = daemon.handle( HuginnMindRequest::Query { instance: slug(INSTANCE), selection: instances.clone(), semantic: None }, now(), - ) else { + ).answered() else { panic!("expected a page"); }; assert_eq!(page.matched, 1); @@ -421,13 +541,13 @@ pub(crate) mod tests { let id = writes[0].clone(); let whole = Selection { projection: "document".into(), ..instances }; let HuginnMindResponse::Query(page) = - daemon.handle(HuginnMindRequest::Query { instance: slug(INSTANCE), selection: whole, semantic: None }, now()) + daemon.handle(HuginnMindRequest::Query { instance: slug(INSTANCE), selection: whole, semantic: None }, now()).answered() else { panic!("expected a page"); }; let PipelinePageItems::Documents(views) = &page.items else { panic!("the document projection is the view") }; assert_eq!(views[0].status, PipelineStatus::InForce); - let view = daemon.handle(HuginnMindRequest::View { instance: slug(INSTANCE), id: id.clone() }, now()); + let view = daemon.handle(HuginnMindRequest::View { instance: slug(INSTANCE), id: id.clone() }, now()).answered(); assert_eq!(view, HuginnMindResponse::View(Some(views[0].clone()))); } @@ -437,18 +557,18 @@ pub(crate) mod tests { fn a_read_or_a_write_naming_another_instance_is_refused_by_the_mind_and_writes_nothing() { let root = tempfile::tempdir().unwrap(); let mut daemon = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); - let landed = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); + let landed = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()).answered(); let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = landed else { panic!("expected a commit"); }; let foreign = MindRefusal::ForeignInstance { declared: OTHER.into(), mind: INSTANCE.into() }; let query = HuginnMindRequest::Query { instance: slug(OTHER), selection: Selection::default(), semantic: None }; - assert_eq!(daemon.handle(query, now()), HuginnMindResponse::Refused(foreign.clone())); + assert_eq!(daemon.handle(query, now()).answered(), HuginnMindResponse::Refused(foreign.clone())); let admit = HuginnMindRequest::Admit(batch(OTHER, vec![identity(OTHER)])); assert_eq!( - daemon.handle(admit, now()), + daemon.handle(admit, now()).answered(), HuginnMindResponse::Admit(PipelineAdmissionOutcome::Refused(foreign.clone())) ); @@ -459,12 +579,12 @@ pub(crate) mod tests { let (question, _) = question_and_ruling(); let admit = HuginnMindRequest::Admit(batch(OTHER, vec![question])); assert_eq!( - daemon.handle(admit, now()), + daemon.handle(admit, now()).answered(), HuginnMindResponse::Admit(PipelineAdmissionOutcome::Refused(foreign.clone())) ); let view = HuginnMindRequest::View { instance: slug(OTHER), id: writes[0].clone() }; - assert_eq!(daemon.handle(view, now()), HuginnMindResponse::Refused(foreign)); + assert_eq!(daemon.handle(view, now()).answered(), HuginnMindResponse::Refused(foreign)); // Three names a comparison could mistake for this mind's. `NEAR` is // `INSTANCE`'s own length and differs in its last byte, so a check @@ -479,12 +599,12 @@ pub(crate) mod tests { for declared in [NEAR, PREFIXED, CASED] { let refusal = MindRefusal::ForeignInstance { declared: declared.into(), mind: INSTANCE.into() }; let query = HuginnMindRequest::Query { instance: slug(declared), selection: Selection::default(), semantic: None }; - assert_eq!(daemon.handle(query, now()), HuginnMindResponse::Refused(refusal.clone()), "{declared}"); + assert_eq!(daemon.handle(query, now()).answered(), HuginnMindResponse::Refused(refusal.clone()), "{declared}"); let view = HuginnMindRequest::View { instance: slug(declared), id: writes[0].clone() }; - assert_eq!(daemon.handle(view, now()), HuginnMindResponse::Refused(refusal.clone()), "{declared}"); + assert_eq!(daemon.handle(view, now()).answered(), HuginnMindResponse::Refused(refusal.clone()), "{declared}"); let admit = HuginnMindRequest::Admit(batch(declared, vec![identity(declared)])); assert_eq!( - daemon.handle(admit, now()), + daemon.handle(admit, now()).answered(), HuginnMindResponse::Admit(PipelineAdmissionOutcome::Refused(refusal)), "{declared}" ); @@ -514,13 +634,13 @@ pub(crate) mod tests { }); let query = HuginnMindRequest::Query { instance: slug(fullwidth), selection: Selection::default(), semantic: None }; - assert_eq!(daemon.handle(query, now()), HuginnMindResponse::Refused(expected.clone())); + assert_eq!(daemon.handle(query, now()).answered(), HuginnMindResponse::Refused(expected.clone())); let view = HuginnMindRequest::View { instance: slug(fullwidth), id: PipelineRef { kind: PipelineKind::Question, id: Short("not-reached".into()) }, }; - assert_eq!(daemon.handle(view, now()), HuginnMindResponse::Refused(expected)); + assert_eq!(daemon.handle(view, now()).answered(), HuginnMindResponse::Refused(expected)); } /// `Admit` is excluded from the daemon's own pre-check (its batch is not a @@ -545,12 +665,12 @@ pub(crate) mod tests { let admit = HuginnMindRequest::Admit(batch(fullwidth, vec![identity(INSTANCE)])); assert_eq!( - daemon.handle(admit, now()), + daemon.handle(admit, now()).answered(), HuginnMindResponse::Admit(PipelineAdmissionOutcome::Refused(expected.clone())) ); let query = HuginnMindRequest::Query { instance: slug(fullwidth), selection: Selection::default(), semantic: None }; - assert_eq!(daemon.handle(query, now()), HuginnMindResponse::Refused(expected)); + assert_eq!(daemon.handle(query, now()).answered(), HuginnMindResponse::Refused(expected)); } /// A refusal a read method raised is the answer the dispatch returns, whole. @@ -572,17 +692,7 @@ pub(crate) mod tests { value: "not a reference".into(), }); let view = HuginnMindRequest::View { instance: slug(INSTANCE), id: invalid }; - assert_eq!(daemon.handle(view, now()), HuginnMindResponse::Refused(malformed)); - - // A semantic query is unavailable until Cut 11 wires an index, and the - // detail is the mind's own sentence. - let semantic = Some(SemanticQuery { text: "who owns the state".into(), top_k: 4 }); - let unwired = - MindRefusal::Unavailable { detail: "semantic query: the index is not wired (Cut 11)".into() }; - assert_eq!( - daemon.handle(HuginnMindRequest::Query { instance: slug(INSTANCE), selection: Selection::default(), semantic }, now()), - HuginnMindResponse::Refused(unwired) - ); + assert_eq!(daemon.handle(view, now()).answered(), HuginnMindResponse::Refused(malformed)); // And the same reads answer rather than refuse when they are asked // something answerable, so the arms above are refusing on the mind's @@ -590,9 +700,9 @@ pub(crate) mod tests { let absent = PipelineRef { kind: PipelineKind::Question, id: Short(format!("{CAMPAIGN}:question:Q1")) }; let present = HuginnMindRequest::View { instance: slug(INSTANCE), id: absent }; - assert_eq!(daemon.handle(present, now()), HuginnMindResponse::View(None)); + assert_eq!(daemon.handle(present, now()).answered(), HuginnMindResponse::View(None)); let plain = HuginnMindRequest::Query { instance: slug(INSTANCE), selection: Selection::default(), semantic: None }; - let HuginnMindResponse::Query(page) = daemon.handle(plain, now()) else { panic!("expected a page") }; + let HuginnMindResponse::Query(page) = daemon.handle(plain, now()).answered() else { panic!("expected a page") }; assert_eq!(page.matched, 1); } @@ -604,7 +714,7 @@ pub(crate) mod tests { fn a_selection_refusal_crosses_the_dispatch_as_itself() { let (_root, mut daemon) = seeded(); let ask = |daemon: &mut Daemon, selection: Selection| { - daemon.handle(HuginnMindRequest::Query { instance: slug(INSTANCE), selection, semantic: None }, now()) + daemon.handle(HuginnMindRequest::Query { instance: slug(INSTANCE), selection, semantic: None }, now()).answered() }; let predicate = |index: &str, value: &str| Selection { fields: Some(vec![cultnet_rs::FieldPredicate { @@ -641,7 +751,7 @@ pub(crate) mod tests { let mut daemon = daemon.with_index(recording.clone()); let (question, ruling) = question_and_ruling(); - let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()); + let outcome = daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()).answered(); let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = outcome else { panic!("expected a commit"); }; @@ -657,13 +767,13 @@ pub(crate) mod tests { let mut daemon = daemon.with_index(FailingIndex); let root = tempfile::tempdir().unwrap(); let mut fresh = open_unindexed(root.path(), &slug(INSTANCE)).unwrap().with_index(FailingIndex); - fresh.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); - let outcome = fresh.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()); + fresh.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()).answered(); + let outcome = fresh.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()).answered(); let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = outcome else { panic!("a failing index does not refuse the batch"); }; for write in &writes { - let view = fresh.handle(HuginnMindRequest::View { instance: slug(INSTANCE), id: write.clone() }, now()); + let view = fresh.handle(HuginnMindRequest::View { instance: slug(INSTANCE), id: write.clone() }, now()).answered(); assert!(matches!(view, HuginnMindResponse::View(Some(_))), "{write:?} is readable"); } assert_eq!(status(&mut daemon).documents, 4, "identity, question, ruling and the derived resolution"); @@ -684,7 +794,7 @@ pub(crate) mod tests { drop(held); let mut planted = open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); - planted.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()); + planted.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])), now()).answered(); drop(planted); let moved = Mind::::store_path_for(root.path(), &slug(OTHER)).unwrap(); std::fs::create_dir_all(moved.parent().unwrap()).unwrap(); diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs index c63041d..d7e4eee 100644 --- a/crates/huginn-daemon/src/index/fakes.rs +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -5,9 +5,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Condvar, Mutex}; -use anyhow::{Result, bail}; +use anyhow::{Result, bail, ensure}; -use super::{CollectionMeta, Described, Embedder, ModelIdentity, Point, VectorIndex}; +use super::{CollectionMeta, Described, Embedder, Hit, ModelIdentity, Point, VectorIndex}; pub(crate) fn identity(digest: &str) -> ModelIdentity { ModelIdentity { name: "model".into(), digest: digest.into(), dimensions: 4 } @@ -30,6 +30,13 @@ impl Gate { } } + /// Shuts the door again and forgets who arrived, so a test that let start-up + /// work through can hold the next call. + pub(crate) fn shut_again(&self) { + let mut state = self.0.0.lock().unwrap(); + *state = (0, false); + } + pub(crate) fn open(&self) { self.0.0.lock().unwrap().1 = true; self.0.1.notify_all(); @@ -109,6 +116,11 @@ pub(crate) struct IndexState { pub(crate) recreated: Vec<(String, CollectionMeta)>, /// The point ids of every upsert call, in order. pub(crate) upserts: Vec>, + /// What every search answers, whatever the vector: the test decides the + /// candidates, as a real index would decide them from vectors. + pub(crate) hits: Vec, + /// The collection, vector length and limit of every search, in order. + pub(crate) searches: Vec<(String, usize, u32)>, } #[derive(Clone, Default)] @@ -169,6 +181,16 @@ impl VectorIndex for FakeIndex { } Ok(()) } + + fn search(&mut self, collection: &str, vector: &[f32], limit: u32) -> Result> { + let mut state = self.state.lock().unwrap(); + if state.down { + bail!("the vector store is down"); + } + state.searches.push((collection.into(), vector.len(), limit)); + ensure!(state.collections.contains_key(collection), "no collection {collection}"); + Ok(state.hits.clone()) + } } /// A working embedder and vector store, for tests whose subject is elsewhere. diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs index ab864c1..9a943ba 100644 --- a/crates/huginn-daemon/src/index/mod.rs +++ b/crates/huginn-daemon/src/index/mod.rs @@ -21,7 +21,17 @@ //! Reconciliation is not a startup event. After any failed step the worker //! reconciles again before it writes anything else: a collection lost while the //! daemon runs is recreated and refilled, and a model re-pulled with other -//! dimensions rebuilds the collection, all without a restart. Refilling needs +//! dimensions rebuilds the collection, all without a restart. Two checks keep +//! that true without a failure to announce it. Before every flush the worker +//! re-reads the model's identity (name, digest, dimensions; about 0.3 s against +//! Ollama) and, if it is not the one the collection was reconciled under, +//! forgets the reconciliation instead of writing: a model re-pulled under the +//! same dimensions never mixes its vectors with the old model's. An idle +//! worker forgets the reconciliation every `Backoff::recheck` (60 s by +//! default), which re-reads the model, describes the collection and lists its +//! points: a collection lost while nothing was being written is found within +//! that interval, and until then `Current` is what the worker last verified. +//! There is one judge either way: `reconcile`. Refilling needs //! the text, so the projector keeps every entry it was given (`known`) beside //! the ids it still owes (`wanted`): the whole indexable mind's text is held in //! memory for the process's life. @@ -45,10 +55,14 @@ //! fail the whole batch forever on one dense document, so it is not set. //! There is no chunking. //! - Points are never deleted. Withdrawn and superseded documents are indexed -//! on purpose and the payload carries no status. Cut 11b's semantic read must -//! therefore join every hit back through the mind: drop ids the mind does not -//! hold, and filter by resolution using RS-3's in-force derivation through -//! selection, never by anything in the payload. +//! on purpose and the payload carries no status. The semantic read therefore +//! takes only candidate ids and scores from a search: `Daemon::finish` joins +//! them back through `Mind::rank`, which drops ids the mind does not hold and +//! lets the selection decide the rest, in-force derivation included. +//! - A search runs on the worker between batches, so it waits for the batch in +//! flight. The serve loop never waits: it holds the reply and answers when +//! the worker has (`ServeOptions::search_timeout` bounds the wait). A search +//! is refused at once while the worker is `Failing` or `Refused`. pub mod ollama; pub mod qdrant; @@ -56,17 +70,18 @@ pub mod qdrant; #[cfg(test)] pub(crate) mod fakes; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::sync::{Arc, Condvar, Mutex, MutexGuard}; use std::time::{Duration, Instant}; -use anyhow::{Context, Result, ensure}; -use huginn_mind::epiphany_pipeline::{PipelineRef, Slug}; +use anyhow::{Context, Result, bail, ensure}; +use huginn_mind::epiphany_pipeline::{PipelineKind, PipelineRef, Short, Slug}; use huginn_mind::wire::IndexStatus; use huginn_mind::{INDEX_TEXT_VERSION, IndexEntry, Mind, MindStore}; use sha2::{Digest, Sha256}; -use crate::daemon::IndexSink; +use crate::daemon::{Hits, IndexSink, SearchTicket}; +use ollama::QUERY_INSTRUCTION; /// The `managed_by` a collection carries when this organ made it. pub const MANAGED_BY: &str = "huginn"; @@ -74,6 +89,9 @@ pub const MANAGED_BY: &str = "huginn"; /// Documents embedded and written per call. pub const BATCH: usize = 32; +/// The most candidates one search asks the index for. +pub const OVERSAMPLE_MAX: u32 = 200; + /// What an embedder is: the model by name and by digest, and the length of the /// vectors it returns. Compatibility of a collection is decided on this and /// on nothing else: never on the address the embedder was reached at. @@ -123,6 +141,16 @@ pub struct Point { pub payload: PointPayload, } +/// One candidate a vector search returned: the document it names, by the id and +/// kind the payload carries, and how near it scored. Nothing else about the +/// document is the index's to say. +#[derive(Clone, Debug, PartialEq)] +pub struct Hit { + pub doc_id: String, + pub kind: String, + pub score: f32, +} + pub trait Embedder { fn model_identity(&mut self) -> Result; /// One vector per text, in order. @@ -137,6 +165,9 @@ pub trait VectorIndex { /// Every point id the collection holds. fn ids(&mut self, collection: &str) -> Result>; fn upsert(&mut self, collection: &str, points: &[Point]) -> Result<()>; + /// The `limit` nearest points to `vector`, best first, with no filter: which + /// documents may be shown is the mind's to decide. + fn search(&mut self, collection: &str, vector: &[f32], limit: u32) -> Result>; } /// The mind's collection: one per instance. @@ -201,7 +232,9 @@ pub struct Projector { known: BTreeMap, /// The ids in `known` whose points the collection may lack. wanted: BTreeSet, - dimensions: Option, + /// The model the collection was reconciled under; `None` until it is, and + /// again whenever that reconciliation is forgotten. + reconciled: Option, } impl Projector { @@ -214,7 +247,7 @@ impl Projector { collection: collection_name(instance), known: BTreeMap::new(), wanted: BTreeSet::new(), - dimensions: None, + reconciled: None, } } @@ -246,26 +279,56 @@ impl Projector { pub fn advance(&mut self) -> Result { let stepped = self.step(); if stepped.is_err() { - self.dimensions = None; + self.reconciled = None; } stepped } + /// `advance` after forgetting the reconciliation on purpose: the idle + /// worker's periodic look at whether the model and the collection are still + /// what it last verified. + pub fn recheck(&mut self) -> Result { + self.reconciled = None; + self.advance() + } + + /// Whether the mind has an identity and the collection has not been + /// reconciled under the model: `advance` has reconciling to do. + pub fn stale(&self) -> bool { + self.mind.is_some() && self.reconciled.is_none() + } + fn step(&mut self) -> Result { if self.mind.is_none() { return Ok(Advance::Idle); } - if self.dimensions.is_none() { + if self.reconciled.is_none() { self.reconcile()?; return Ok(Advance::Progressed); } if self.wanted.is_empty() { return Ok(Advance::Idle); } + if !self.model_unchanged()? { + return Ok(Advance::Progressed); + } self.flush_batch()?; Ok(Advance::Progressed) } + /// Re-reads the model's identity and compares it with the one the + /// collection was reconciled under. A different model, digest or length + /// forgets the reconciliation, so the next step reconciles (and rebuilds) + /// instead of writing or searching with vectors the collection cannot mix. + fn model_unchanged(&mut self) -> Result { + let now = self.embedder.model_identity().context("re-reading the embedding model's identity")?; + if self.reconciled.as_ref() == Some(&now) { + return Ok(true); + } + self.reconciled = None; + Ok(false) + } + /// The only place a point is judged missing. Reads the model identity, /// makes the collection agree with it (creating it, or rebuilding it when /// the model, its digest, its dimensions or the text version differ), @@ -306,7 +369,7 @@ impl Projector { } let present = self.index.ids(&self.collection).context("listing the collection's points")?; self.wanted = self.known.keys().filter(|document_id| !present.contains(&point_id(document_id))).cloned().collect(); - self.dimensions = Some(identity.dimensions); + self.reconciled = Some(identity); Ok(()) } @@ -322,7 +385,7 @@ impl Projector { } fn flush(&mut self) -> Result<()> { - let dimensions = self.dimensions.context("flush before reconcile")? as usize; + let dimensions = self.reconciled.as_ref().context("flush before reconcile")?.dimensions as usize; let batch: Vec = self.wanted.iter().take(BATCH).filter_map(|document_id| self.known.get(document_id)).cloned().collect(); if batch.is_empty() { @@ -358,9 +421,40 @@ impl Projector { Ok(()) } + /// The candidates a query text finds: the text is embedded with the model's + /// query instruction and the collection is searched, with no filter and an + /// oversample of `min(top_k * 4, 200)`, because the mind will drop most of + /// what the index cannot know is not in force. A model that has changed + /// since the reconciliation refuses the search rather than compare vectors + /// of two models. A hit naming a kind this organ has no name for is logged + /// and left out. + pub fn search(&mut self, text: &str, top_k: u32) -> Result { + let Some(dimensions) = self.reconciled.as_ref().map(|model| model.dimensions as usize) else { + bail!("the index has not been reconciled with the model and the collection"); + }; + ensure!( + self.model_unchanged().map_err(|error| anyhow::anyhow!("{error}"))?, + "the embedding model changed since the collection was built; the index is being rebuilt for it" + ); + let query = format!("Instruct: {QUERY_INSTRUCTION}\nQuery: {text}"); + let mut vectors = self.embedder.embed(&[query]).context("embedding the query")?; + ensure!(vectors.len() == 1, "the embedder returned {} vectors for one query", vectors.len()); + let vector = vectors.remove(0); + ensure!(vector.len() == dimensions, "the embedder returned a query vector of {} where the collection holds {dimensions}", vector.len()); + let found = self.index.search(&self.collection, &vector, top_k.saturating_mul(4).min(OVERSAMPLE_MAX))?; + let mut hits = Vec::with_capacity(found.len()); + for hit in found { + match PipelineKind::ALL.into_iter().find(|kind| kind.name() == hit.kind) { + Some(kind) => hits.push((PipelineRef { kind: *kind, id: Short(hit.doc_id) }, hit.score)), + None => eprintln!("huginn: the index returned {} under the unknown kind {:?}; it is left out", hit.doc_id, hit.kind), + } + } + Ok(hits) + } + /// The health after a step that did not fail. pub fn progress_status(&self) -> IndexStatus { - match (self.dimensions, self.pending()) { + match (&self.reconciled, self.pending()) { (None, 0) if self.mind.is_none() => IndexStatus::Current, (None, pending) => IndexStatus::Reconciling { pending }, (Some(_), 0) => IndexStatus::Current, @@ -379,17 +473,19 @@ impl Projector { } } -/// The retry delay: it starts at `initial`, doubles per consecutive failure and -/// stops at `max`. +/// The worker's clock: the retry delay starts at `initial`, doubles per +/// consecutive failure and stops at `max`; an idle worker verifies the model +/// and the collection again every `recheck`. #[derive(Clone, Copy, Debug)] pub struct Backoff { pub initial: Duration, pub max: Duration, + pub recheck: Duration, } impl Default for Backoff { fn default() -> Self { - Self { initial: Duration::from_secs(30), max: Duration::from_secs(600) } + Self { initial: Duration::from_secs(30), max: Duration::from_secs(600), recheck: Duration::from_secs(60) } } } @@ -405,6 +501,11 @@ struct Shared { mind: Option, status: IndexStatus, closed: bool, + /// Searches started and not yet taken by the worker. + searches: VecDeque<(SearchTicket, String, u32)>, + /// Searches the worker finished and the serve loop has not collected. + found: Vec<(SearchTicket, Result)>, + next_ticket: u64, } type Lock = Arc<(Mutex, Condvar)>; @@ -413,27 +514,36 @@ fn lock(shared: &Lock) -> MutexGuard<'_, Shared> { shared.0.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) } -/// With no timeout, blocks until the inbox holds something, the mind reveals -/// its identity, or the sink is closed. With one, sleeps out the backoff whatever arrives: a failing index -/// is retried on its schedule, not once per admission. -fn wait_for_work(shared: &Lock, timeout: Option) { +/// How long the worker waits, and what ends the wait early. +enum Wait { + /// Idle: until something arrives (an entry, the mind's identity, a search) + /// or `recheck` passes. + Work { recheck: Duration }, + /// Backing off from a failure: the whole delay, whatever arrives. A failing + /// index is retried on its schedule, not once per admission. + Backoff(Duration), +} + +/// Whether the wait ended because the recheck interval passed. +fn wait_for_work(shared: &Lock, wait: Wait) -> bool { + let (timeout, wakes_on_work) = match wait { + Wait::Work { recheck } => (recheck, true), + Wait::Backoff(delay) => (delay, false), + }; + let deadline = Instant::now() + timeout; let mut guard = lock(shared); - match timeout { - None => { - while guard.inbox.is_empty() && guard.mind.is_none() && !guard.closed { - guard = shared.1.wait(guard).unwrap_or_else(|poisoned| poisoned.into_inner()); - } + loop { + if guard.closed { + return false; } - Some(timeout) => { - let deadline = Instant::now() + timeout; - while !guard.closed { - let left = deadline.saturating_duration_since(Instant::now()); - if left.is_zero() { - return; - } - guard = shared.1.wait_timeout(guard, left).unwrap_or_else(|poisoned| poisoned.into_inner()).0; - } + if wakes_on_work && (!guard.inbox.is_empty() || guard.mind.is_some() || !guard.searches.is_empty()) { + return false; } + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return wakes_on_work; + } + guard = shared.1.wait_timeout(guard, left).unwrap_or_else(|poisoned| poisoned.into_inner()).0; } } @@ -460,37 +570,72 @@ fn including(status: &IndexStatus, queued: u32) -> IndexStatus { } } +/// Runs every search the serve loop has queued, one at a time, and leaves each +/// answer for the loop to collect. Between batches, never inside one. +fn serve_searches(projector: &mut Projector, shared: &Lock) { + loop { + let Some((ticket, text, top_k)) = lock(shared).searches.pop_front() else { return }; + let found = projector.search(&text, top_k).map_err(|error| format!("{error:#}")); + lock(shared).found.push((ticket, found)); + } +} + +/// Answers every queued search with why the index cannot: they were accepted +/// while it was healthy and it has failed since. +fn refuse_searches(shared: &Lock, why: &str) { + let mut guard = lock(shared); + while let Some((ticket, ..)) = guard.searches.pop_front() { + guard.found.push((ticket, Err(why.to_owned()))); + } +} + fn work(mut projector: Projector, shared: Lock, backoff: Backoff) { let mut failures = 0_u32; let mut delay = backoff.initial; + let mut recheck = false; loop { - let (mind, inbox) = { + // The inbox is taken, absorbed and its effect published under one + // lock: between taking entries out of the inbox and publishing them + // as wanted, `status` would otherwise read a stale `Current` with + // nothing queued, and a reader would see the mind caught up while + // entries were in the worker's hands. + { let mut guard = lock(&shared); if guard.closed { return; } - (guard.mind.take(), std::mem::take(&mut guard.inbox)) - }; - if let Some(mind) = mind { - projector.identify(mind); + let mind = guard.mind.take(); + let inbox = std::mem::take(&mut guard.inbox); + if let Some(mind) = mind { + projector.identify(mind); + } + projector.want(inbox); + // While a failure is being reported, the report stands until an + // attempt succeeds: publishing progress between retries would + // flicker. + if failures == 0 { + guard.status = projector.progress_status(); + } } - projector.want(inbox); - // While a failure is being reported, the report stands until an - // attempt succeeds: publishing progress between retries would flicker. - if failures == 0 { - publish(&shared, projector.progress_status()); - } - match projector.advance() { + shared.1.notify_all(); + let stepped = if std::mem::take(&mut recheck) { projector.recheck() } else { projector.advance() }; + match stepped { Ok(Advance::Progressed) => { failures = 0; delay = backoff.initial; publish(&shared, projector.progress_status()); + serve_searches(&mut projector, &shared); } Ok(Advance::Idle) => { failures = 0; delay = backoff.initial; publish(&shared, projector.progress_status()); - wait_for_work(&shared, None); + serve_searches(&mut projector, &shared); + // A search that found the model changed has left the + // reconciliation to redo: do it now, not at the next wake. + if !projector.stale() { + recheck = wait_for_work(&shared, Wait::Work { recheck: backoff.recheck }); + } } Err(error) => { failures = failures.saturating_add(1); @@ -500,7 +645,8 @@ fn work(mut projector: Projector, shared: Loc delay ); publish(&shared, projector.failure_status(&error, failures)); - wait_for_work(&shared, Some(delay)); + refuse_searches(&shared, &format!("the index could not advance: {error}")); + wait_for_work(&shared, Wait::Backoff(delay)); delay = backoff.after(delay); } } @@ -538,7 +684,15 @@ impl WorkerSink { let mut projector = Projector::new(embedder, index, instance, mind); projector.want(startup); let shared: Lock = Arc::new(( - Mutex::new(Shared { inbox: Vec::new(), mind: None, status: projector.progress_status(), closed: false }), + Mutex::new(Shared { + inbox: Vec::new(), + mind: None, + status: projector.progress_status(), + closed: false, + searches: VecDeque::new(), + found: Vec::new(), + next_ticket: 0, + }), Condvar::new(), )); let worker = Arc::clone(&shared); @@ -596,6 +750,27 @@ impl IndexSink for WorkerSink { let guard = lock(&self.shared); including(&guard.status, u32::try_from(guard.inbox.len()).unwrap_or(u32::MAX)) } + + /// Queues the search for the worker and returns its ticket. A worker that + /// is failing or refused is not asked: its own report is the refusal. + fn search(&mut self, text: &str, top_k: u32) -> Result { + let mut guard = lock(&self.shared); + match &guard.status { + IndexStatus::Failing { error, .. } => return Err(format!("the semantic index is failing: {error}")), + IndexStatus::Refused { reason, .. } => return Err(format!("the semantic index is refused: {reason}")), + _ => {} + } + let ticket = SearchTicket(guard.next_ticket); + guard.next_ticket += 1; + guard.searches.push_back((ticket, text.to_owned(), top_k)); + drop(guard); + self.shared.1.notify_all(); + Ok(ticket) + } + + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { + std::mem::take(&mut lock(&self.shared).found) + } } /// Plain HTTP to one base address, in text: the adapters own what the text diff --git a/crates/huginn-daemon/src/index/qdrant.rs b/crates/huginn-daemon/src/index/qdrant.rs index a794ac1..2999220 100644 --- a/crates/huginn-daemon/src/index/qdrant.rs +++ b/crates/huginn-daemon/src/index/qdrant.rs @@ -7,7 +7,7 @@ use std::time::Duration; use anyhow::{Context, Result, ensure}; use serde_json::{Value, json}; -use super::{CollectionMeta, Described, Http, Method, Point, VectorIndex}; +use super::{CollectionMeta, Described, Hit, Http, Method, Point, VectorIndex}; /// One Qdrant. Every path names a collection this organ was told to own; it /// never lists or touches another. @@ -132,4 +132,20 @@ impl VectorIndex for QdrantIndex { )?; Ok(()) } + + fn search(&mut self, collection: &str, vector: &[f32], limit: u32) -> Result> { + let body = json!({ "vector": vector, "limit": limit, "with_payload": ["doc_id", "kind"], "with_vector": false }); + let (_, answer) = self.ask(Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?; + let found = answer["result"].as_array().context("Qdrant's search has no result")?; + found + .iter() + .map(|point| { + Ok(Hit { + doc_id: point["payload"]["doc_id"].as_str().context("a hit carries no doc_id")?.to_owned(), + kind: point["payload"]["kind"].as_str().context("a hit carries no kind")?.to_owned(), + score: point["score"].as_f64().context("a hit carries no score")? as f32, + }) + }) + .collect() + } } diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index 90524b8..e0e4ecf 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -246,7 +246,7 @@ fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { } fn whoami_index(daemon: &mut Daemon) -> IndexStatus { - match daemon.handle(HuginnMindRequest::Whoami, now()) { + match daemon.handle(HuginnMindRequest::Whoami, now()).answered() { HuginnMindResponse::Whoami(status) => status.index, other => panic!("expected a status, got {other:?}"), } @@ -260,7 +260,7 @@ fn committed(response: HuginnMindResponse) { } fn quick() -> Backoff { - Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2) } + Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600) } } /// The index never delays or refuses admission: the embedder is shut, the @@ -278,10 +278,10 @@ fn an_admission_replies_while_the_embedder_is_blocked() { let mut daemon = Daemon::new(mind, sink); daemon.index().wait_status(|status| *status == IndexStatus::Current); - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, campaign_seed())), now())); + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, campaign_seed())), now()).answered()); gate.wait_arrived(1); let (question, ruling) = question_and_ruling(); - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now())); + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()).answered()); assert_eq!( whoami_index(&mut daemon), IndexStatus::Behind { pending: 4 }, @@ -309,7 +309,7 @@ fn a_failing_embedder_is_visible_in_whoami_and_retried_never_dropped() { embedder.state.lock().unwrap().down = true; let (question, ruling) = question_and_ruling(); for documents in [campaign_seed(), vec![question, ruling]] { - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), now())); + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), now()).answered()); } let seen = daemon.index().wait_status(|status| matches!(status, IndexStatus::Failing { attempts, .. } if *attempts >= 2)); let IndexStatus::Failing { pending, error, .. } = seen else { panic!() }; @@ -627,6 +627,7 @@ fn dropping_the_sink_stops_the_worker() { fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() { let backoff = Backoff::default(); assert_eq!((backoff.initial, backoff.max), (Duration::from_secs(30), Duration::from_secs(600))); + assert_eq!(backoff.recheck, Duration::from_secs(60), "an idle worker verifies the model and the collection each minute"); let mut delay = backoff.initial; let mut seen = vec![delay.as_secs()]; for _ in 0..7 { @@ -637,7 +638,7 @@ fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() { seen.push(delay.as_secs()); } assert_eq!(seen, [30, 60, 120, 240, 480, 600, 600, 600]); - let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3) }; + let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3), recheck: Duration::from_secs(3600) }; assert_eq!(small.after(Duration::from_millis(1)), Duration::from_millis(2)); assert_eq!(small.after(Duration::from_millis(2)), Duration::from_millis(3)); } @@ -666,8 +667,9 @@ fn a_collection_lost_after_startup_is_recreated_and_refilled() { assert_eq!(projector.progress_status(), IndexStatus::Current); } -/// F2. A model re-pulled with other dimensions fails one write, then the next -/// attempt reads the new identity and rebuilds the collection to match. +/// F2. A model re-pulled with other dimensions is seen before anything is +/// written: no vector of the new length is offered to the old collection, and +/// the next step rebuilds the collection to match. #[test] fn a_model_that_changes_dimensions_after_startup_rebuilds_the_collection() { let (_root, mind) = mind_with_documents(); @@ -677,7 +679,9 @@ fn a_model_that_changes_dimensions_after_startup_rebuilds_the_collection() { embedder.state.lock().unwrap().identity.dimensions = 8; projector.want(entries[3..].iter().cloned()); - assert!(projector.advance().is_err(), "a vector of 8 does not fit a collection of 4"); + let writes = index.state.lock().unwrap().upserts.len(); + assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the change is seen, not written through"); + assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "no vector of 8 was offered to a collection of 4"); drain(&mut projector); let state = index.state.lock().unwrap(); @@ -740,7 +744,7 @@ fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { let mut daemon = Daemon::new(mind, sink); let seed = campaign_seed(); let without_text = vec![seed[0].clone(), seed[1].clone()]; - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, without_text)), now())); + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, without_text)), now()).answered()); for _ in 0..500 { if index.state.lock().unwrap().collections.contains_key(&collection()) { break; @@ -751,3 +755,234 @@ fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { let Described::Labelled(label) = &state.collections[&collection()].label else { panic!("no collection was made") }; assert_eq!(Some(&label.mind), daemon.mind().genesis_receipt_id().unwrap().as_ref()); } + +/// Waits for a condition the worker will make true, without a clock in the +/// assertion: it polls for at most ten seconds and then says what never +/// happened. +fn eventually(what: &str, condition: impl Fn() -> bool) { + for _ in 0..2000 { + if condition() { + return; + } + std::thread::sleep(Duration::from_millis(5)); + } + panic!("{what} never happened"); +} + +/// A model re-pulled under the same name and dimensions but a new digest must +/// not mix its vectors with the old model's. The check is before the write, +/// not after a failure: nothing is offered to the old collection, the +/// collection is rebuilt under the new digest, and every document is embedded +/// again. +#[test] +fn a_model_re_pulled_with_the_same_dimensions_never_writes_beside_the_old_vectors() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + + embedder.state.lock().unwrap().identity.digest = "d2".into(); + projector.want(entries[3..].iter().cloned()); + let writes = index.state.lock().unwrap().upserts.len(); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "the new vector was not written into the old collection"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 1 }, "the reconciliation was forgotten, not the entry"); + + drain(&mut projector); + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2); + assert_eq!(state.recreated.last().unwrap().1.model_digest, "d2"); + let stored = &state.collections[&collection()]; + assert_eq!(stored.points.len(), 4, "every document is in the rebuilt collection"); + assert_eq!(embedder.texts().len(), 3 + 4, "and was embedded again under the new digest"); +} + +/// Nothing is written while idle, so no write can fail to announce that the +/// collection is gone or the model has changed. The idle worker looks again on +/// its own clock: a lost collection is refilled and a re-pulled model rebuilds +/// it, with no admission to prompt either. +#[test] +fn an_idle_worker_finds_a_lost_collection_and_a_changed_model_without_a_write() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let recheck = Backoff { recheck: Duration::from_millis(20), ..quick() }; + let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries.clone(), recheck); + sink.wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()), point_ids(&entries)); + + index.state.lock().unwrap().collections.remove(&collection()); + eventually("the lost collection being made again and refilled", || { + index.state.lock().unwrap().recreated.len() == 2 && index.holding(&collection()) == point_ids(&entries) + }); + + embedder.state.lock().unwrap().identity.digest = "d2".into(); + eventually("the collection being rebuilt under the new digest", || { + let state = index.state.lock().unwrap(); + state.recreated.len() == 3 + && state.recreated.last().is_some_and(|(_, meta)| meta.model_digest == "d2") + && state.collections[&collection()].points.len() == 4 + }); + sink.wait_status(|status| *status == IndexStatus::Current); +} + +/// A search embeds the query behind the model's instruction, asks the index +/// with no filter for `min(top_k * 4, 200)`, and hands back candidates by id +/// and kind; a hit of a kind this organ has no name for is left out. +#[test] +fn a_search_embeds_the_query_with_its_instruction_and_oversamples() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + index.state.lock().unwrap().hits = vec![ + Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }, + Hit { doc_id: "eureka-state:mystery:X".into(), kind: "mystery".into(), score: 0.8 }, + ]; + + let hits = projector.search("who owns the state", 3).unwrap(); + assert_eq!(hits.len(), 1, "the unknown kind is left out"); + assert_eq!(hits[0].0.id.0, "eureka-state:question:Q1"); + assert_eq!(hits[0].0.kind, PipelineKind::Question); + assert_eq!(hits[0].1, 0.9); + let embedded = embedder.texts(); + assert_eq!(embedded.last().unwrap(), &format!("Instruct: {QUERY_INSTRUCTION}\nQuery: who owns the state")); + projector.search("again", 100).unwrap(); + let searches = index.state.lock().unwrap().searches.clone(); + assert_eq!(searches.iter().map(|(name, length, limit)| (name.as_str(), *length, *limit)).collect::>(), [ + (collection().as_str(), 4, 12), + (collection().as_str(), 4, 200), + ]); +} + +/// A search is never run against a collection that is not known to fit the +/// model: before reconciliation, and after the model has changed, it refuses +/// by saying so rather than comparing vectors of two models. +#[test] +fn a_search_refuses_when_the_index_is_not_reconciled_with_the_model() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + assert!(format!("{:#}", fresh.search("anything", 3).unwrap_err()).contains("not been reconciled")); + assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); + + let mut projector = projected(&embedder, &index, &entries); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + let before = embedder.texts().len(); + assert!(format!("{:#}", projector.search("anything", 3).unwrap_err()).contains("model changed")); + assert_eq!(embedder.texts().len(), before, "the query was not embedded under the new model"); + assert!(index.state.lock().unwrap().searches.is_empty()); + assert!(projector.stale(), "and the reconciliation is to be redone"); +} + +/// The search asks for the nearest points to a vector with no filter of any +/// kind (which documents may be shown is the mind's), and reads back the +/// document id, the kind and the score, in the order Qdrant gave them. A hit +/// missing what the join needs is an error, not a guess. +#[test] +fn qdrant_search_sends_no_filter_and_reads_ids_kinds_and_scores() { + let name = collection(); + let answering = stub(&[( + "POST /collections/", + 200, + r#"{"result":[{"id":"p1","score":0.75,"payload":{"doc_id":"eureka-state:question:Q1","kind":"question"}},{"id":"p2","score":0.5,"payload":{"doc_id":"eureka-state:ruling:R1","kind":"ruling"}}]}"#, + )]); + let hits = QdrantIndex::new(&answering.url).search(&name, &[0.5, 1.0, 2.0, 3.0], 12).unwrap(); + assert_eq!( + hits, + [ + Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.75 }, + Hit { doc_id: "eureka-state:ruling:R1".into(), kind: "ruling".into(), score: 0.5 }, + ] + ); + let seen = answering.seen.lock().unwrap(); + assert_eq!((seen[0].0.as_str(), seen[0].1.as_str()), ("POST", format!("/collections/{name}/points/search").as_str())); + for expected in ["\"vector\":[0.5,1.0,2.0,3.0]", "\"limit\":12", "\"with_vector\":false"] { + assert!(seen[0].2.contains(expected), "{expected} in {}", seen[0].2); + } + assert!(!seen[0].2.contains("filter"), "no predicate is lowered into the index: {}", seen[0].2); + drop(seen); + + let nameless = stub(&[("POST /collections/", 200, r#"{"result":[{"id":"p","score":0.5,"payload":{"kind":"question"}}]}"#)]); + assert!(QdrantIndex::new(&nameless.url).search(&name, &[1.0], 1).is_err()); + let failing = stub(&[("POST /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); + assert!(QdrantIndex::new(&failing.url).search(&name, &[1.0], 1).is_err()); +} + +fn shared_with( + inbox: Vec, + mind: Option, + searches: Vec<(SearchTicket, String, u32)>, + closed: bool, +) -> Lock { + Arc::new(( + Mutex::new(Shared { + inbox, + mind, + status: IndexStatus::Current, + closed, + searches: searches.into(), + found: Vec::new(), + next_ticket: 0, + }), + Condvar::new(), + )) +} + +/// The idle wait is for work: an entry, the mind's identity, a search or the +/// sink closing ends it at once, and only a quiet worker waits out the recheck +/// interval, after which it says the interval passed. +#[test] +fn an_idle_wait_ends_for_work_and_otherwise_at_the_recheck() { + let (_root, mind) = mind_with_documents(); + let entry = mind.index_entries(None).unwrap().remove(0); + let waits = |shared: &Lock| { + let started = Instant::now(); + let recheck = wait_for_work(shared, Wait::Work { recheck: Duration::from_secs(30) }); + (recheck, started.elapsed() < Duration::from_secs(10)) + }; + assert_eq!(waits(&shared_with(vec![entry], None, Vec::new(), false)), (false, true), "an entry"); + assert_eq!(waits(&shared_with(Vec::new(), Some(MIND.into()), Vec::new(), false)), (false, true), "the mind's identity"); + assert_eq!(waits(&shared_with(Vec::new(), None, vec![(SearchTicket(0), "q".into(), 3)], false)), (false, true), "a search"); + assert_eq!(waits(&shared_with(Vec::new(), None, Vec::new(), true)), (false, true), "the sink closing"); + + let quiet = shared_with(Vec::new(), None, Vec::new(), false); + let started = Instant::now(); + assert!(wait_for_work(&quiet, Wait::Work { recheck: Duration::from_millis(60) }), "a quiet worker is told the interval passed"); + assert!(started.elapsed() >= Duration::from_millis(60), "and only after it did"); +} + +/// A backoff is the whole delay, whatever arrives, and never reports a +/// recheck: a failing index is retried on its schedule. +#[test] +fn a_backoff_wait_is_the_whole_delay_whatever_arrives() { + let (_root, mind) = mind_with_documents(); + let entry = mind.index_entries(None).unwrap().remove(0); + let busy = shared_with(vec![entry], Some(MIND.into()), vec![(SearchTicket(0), "q".into(), 3)], false); + let started = Instant::now(); + assert!(!wait_for_work(&busy, Wait::Backoff(Duration::from_millis(80)))); + assert!(started.elapsed() >= Duration::from_millis(80), "the delay was waited out although work was queued"); + let closed = shared_with(Vec::new(), None, Vec::new(), true); + let started = Instant::now(); + assert!(!wait_for_work(&closed, Wait::Backoff(Duration::from_secs(30)))); + assert!(started.elapsed() < Duration::from_secs(10), "a closed sink ends even a backoff"); +} + +/// Searches accepted while the index was healthy are answered, with the +/// reason, when the worker fails: nothing is left to wait for its deadline. +#[test] +fn queued_searches_are_answered_with_why_when_the_worker_gives_up() { + let queued = vec![(SearchTicket(3), "a".into(), 1), (SearchTicket(4), "b".into(), 2)]; + let shared = shared_with(Vec::new(), None, queued, false); + refuse_searches(&shared, "the index could not advance: down"); + let guard = lock(&shared); + assert!(guard.searches.is_empty()); + assert_eq!( + guard.found, + [ + (SearchTicket(3), Err("the index could not advance: down".to_string())), + (SearchTicket(4), Err("the index could not advance: down".to_string())), + ] + ); +} diff --git a/crates/huginn-daemon/src/lib.rs b/crates/huginn-daemon/src/lib.rs index fc27e59..a9fbb9f 100644 --- a/crates/huginn-daemon/src/lib.rs +++ b/crates/huginn-daemon/src/lib.rs @@ -21,6 +21,9 @@ pub mod envelope; pub mod index; pub mod serve; -pub use daemon::{Daemon, IndexSink, runtime_id}; +#[cfg(test)] +mod semantic_tests; + +pub use daemon::{Daemon, Handled, Hits, IndexSink, Search, SearchTicket, runtime_id}; pub use envelope::{FAILURE_SCHEMA, OperationFailure}; pub use serve::{Options, ServeOptions, parse_options, run, startup}; diff --git a/crates/huginn-daemon/src/semantic_tests.rs b/crates/huginn-daemon/src/semantic_tests.rs new file mode 100644 index 0000000..311a51e --- /dev/null +++ b/crates/huginn-daemon/src/semantic_tests.rs @@ -0,0 +1,434 @@ +//! The semantic read through the real serve loop: a UDP hub, real sessions, the +//! worker thread, and fake embedder and vector-store ports whose behaviour the +//! test decides. Nothing here calls `Daemon::handle` directly: the rules under +//! test are about what the loop does while an answer is outstanding. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread::JoinHandle; +use std::time::Duration; + +use anyhow::Result; +use cultnet_rs::{CULTNET_OPERATION_CONNECTION_ID, CultMesh, CultMeshRudpSocketOptions, FieldPredicate, Selection}; +use huginn_mind::epiphany_pipeline::{ + Date, PipelineDocument, PipelineKind, PipelineQuestion, PipelineResolution, PipelineRef, PipelineRuling, + QuestionOption, ResolutionOutcome, RulingAuthority, Short, Title, +}; +use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, IndexStatus}; +use huginn_mind::{Mind, MindRefusal, OwnedRedbMessagePackBackingStore, PipelineAdmissionOutcome, PipelinePageItems, SemanticQuery}; +use tempfile::TempDir; + +use crate::daemon::Daemon; +use crate::daemon::tests::{CAMPAIGN, INSTANCE, batch, campaign_seed, now, slug}; +use crate::envelope::{decode_response, encode_request}; +use crate::index::fakes::{FakeEmbedder, FakeIndex, Gate}; +use crate::index::{Backoff, Described, Hit, WorkerSink, collection_name}; +use crate::serve::{ServeOptions, bind, run, schema_registry}; + +type Client = cultnet_rs::CultNetRudpSocketTransportConnection; + +fn question(label: &str) -> PipelineDocument { + PipelineDocument::Question(PipelineQuestion { + campaign: slug(CAMPAIGN), + label: label.into(), + title: Title(format!("Question {label}")), + question: format!("Question {label}?").as_str().into(), + options: vec![ + QuestionOption { label: "A".into(), text: "the organ".into() }, + QuestionOption { label: "B".into(), text: "the client".into() }, + ], + recommended: "A".into(), + depends: vec![], + raised_in: None, + asked_on: Date("2026-09-16".into()), + }) +} + +/// A ruling, answering a question when it names one. +fn ruling(label: &str, answers: Option<&str>) -> PipelineDocument { + PipelineDocument::Ruling(PipelineRuling { + campaign: slug(CAMPAIGN), + label: label.into(), + title: Title(format!("Ruling {label}")), + answers: answers.map(|question| Short(id("question", question))), + choice: answers.map(|_| "A".into()), + ruling: format!("Ruling {label}.").as_str().into(), + operator_quote: None, + ruled_on: Date("2026-09-16".into()), + precedents: vec![], + authority: RulingAuthority::Operator, + }) +} + +fn resolution(subject: PipelineKind, key: &str, outcome: ResolutionOutcome) -> PipelineDocument { + PipelineDocument::Resolution(PipelineResolution { + subject: PipelineRef { kind: subject, id: Short(key.into()) }, + sequence: 1, + outcome, + rationale: "Resolved.".into(), + resolved_on: Date("2026-09-16".into()), + }) +} + +fn id(kind: &str, local: &str) -> String { + format!("{CAMPAIGN}:{kind}:{local}") +} + +fn hit(kind: &str, key: String, score: f32) -> Hit { + Hit { doc_id: key, kind: kind.into(), score } +} + +/// Every way a document stops standing, and two that do: Q1 is answered (so +/// resolved), Q2 was answered and the answer withdrawn (its resolution is the +/// resolved one and the question stands again), R3 is superseded by R4. +fn shelf() -> Vec> { + vec![ + campaign_seed(), + vec![question("Q1"), question("Q2")], + vec![ruling("R1", Some("Q1"))], + vec![ruling("R2", Some("Q2"))], + vec![resolution(PipelineKind::Resolution, &id("resolution", "question.Q2.n1"), ResolutionOutcome::Withdrawn { reason: "moot".into() })], + vec![ruling("R3", None), ruling("R4", None)], + vec![resolution( + PipelineKind::Ruling, + &id("ruling", "R3"), + ResolutionOutcome::Superseded { by: vec![PipelineRef { kind: PipelineKind::Ruling, id: Short(id("ruling", "R4")) }] }, + )], + ] +} + +fn standing() -> Selection { + Selection { + fields: Some(vec![FieldPredicate { + index: "in_force".into(), + op: "any_of".into(), + values: Some(vec!["true".into()]), + number: None, + }]), + ..Selection::default() + } +} + +fn semantic(selection: Selection, top_k: u32) -> HuginnMindRequest { + HuginnMindRequest::Query { + instance: slug(INSTANCE), + selection, + semantic: Some(SemanticQuery { text: "who owns the state".into(), top_k }), + } +} + +/// How many documents a plain query finds. +fn documents(client: &mut Client, message_id: &str) -> usize { + ids(&ask(client, message_id, &plain())).len() +} + +fn plain() -> HuginnMindRequest { + HuginnMindRequest::Query { instance: slug(INSTANCE), selection: Selection::default(), semantic: None } +} + +/// The ids on a page, in its order. +fn ids(response: &HuginnMindResponse) -> Vec { + let HuginnMindResponse::Query(page) = response else { panic!("expected a page, got {response:?}") }; + match &page.items { + PipelinePageItems::Headers(headers) => headers.iter().map(|header| header.id.id.0.clone()).collect(), + PipelinePageItems::Documents(views) => views.iter().map(|view| view.id.id.0.clone()).collect(), + } +} + +fn quick() -> Backoff { + Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600) } +} + +/// A daemon serving a mind over a real socket, with the ports the test gave it. +struct Harness { + endpoint: String, + stopping: Arc, + serving: Option>>, + embedder: FakeEmbedder, + index: FakeIndex, + _root: TempDir, +} + +impl Harness { + fn start(seed: Vec>, embedder: FakeEmbedder, index: FakeIndex, search_timeout: Duration) -> Self { + let root = tempfile::tempdir().unwrap(); + let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + for documents in seed { + let outcome = mind.admit(batch(INSTANCE, documents), now()); + assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); + } + let sink = WorkerSink::spawn( + embedder.clone(), + index.clone(), + &slug(INSTANCE), + mind.genesis_receipt_id().unwrap(), + mind.index_entries(None).unwrap(), + quick(), + ); + let mut daemon: Daemon = Daemon::new(mind, sink); + let mut hub = bind("127.0.0.1:0".parse().unwrap(), &daemon.runtime_id()).unwrap(); + let endpoint = format!("rudp://{}", hub.local_addr().unwrap()); + let registry = schema_registry().unwrap(); + let stopping = Arc::new(AtomicBool::new(false)); + let loop_stopping = Arc::clone(&stopping); + let options = ServeOptions { search_timeout, ..ServeOptions::default() }; + let serving = std::thread::spawn(move || run(&mut daemon, &mut hub, ®istry, &loop_stopping, &options)); + Self { endpoint, stopping, serving: Some(serving), embedder, index, _root: root } + } + + fn client(&self) -> Client { + let mut client = CultMesh::create_rudp_client_for_endpoint( + "semantic-test".to_string(), + CULTNET_OPERATION_CONNECTION_ID, + &self.endpoint, + CultMeshRudpSocketOptions::default(), + ) + .unwrap(); + client.connect(Vec::new()).unwrap(); + for _ in 0..1000 { + let _ = client.receive_once(); + if client.connected() { + return client; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("the client never connected"); + } +} + +impl Drop for Harness { + fn drop(&mut self) { + self.stopping.store(true, Ordering::Relaxed); + // A loop that is stuck waiting on the index is a failure of the test, + // not a reason to hang the run: give it a moment and leave it behind. + if let Some(serving) = self.serving.take() { + for _ in 0..1500 { + if serving.is_finished() { + let _ = serving.join(); + return; + } + std::thread::sleep(Duration::from_millis(2)); + } + } + } +} + +fn send(client: &mut Client, message_id: &str, request: &HuginnMindRequest) { + client.send_schema_message(&encode_request(message_id, request, None).unwrap()).unwrap(); +} + +/// The reply to `message_id` on this client's session, or `None` if none came +/// in `patience` polls of two milliseconds. +fn receive(client: &mut Client, message_id: &str, patience: u32) -> Option { + for _ in 0..patience { + client.poll_resends().unwrap(); + if let Some(reply) = client.receive_schema_message_once().unwrap() { + let (correlated, response) = decode_response(&reply).unwrap(); + assert_eq!(correlated, message_id); + return Some(response.expect("an answer, not an envelope failure")); + } + std::thread::sleep(Duration::from_millis(2)); + } + None +} + +fn ask(client: &mut Client, message_id: &str, request: &HuginnMindRequest) -> HuginnMindResponse { + send(client, message_id, request); + receive(client, message_id, 3000).unwrap_or_else(|| panic!("no reply to {message_id}")) +} + +fn index_status(client: &mut Client, message_id: &str) -> IndexStatus { + match ask(client, message_id, &HuginnMindRequest::Whoami) { + HuginnMindResponse::Whoami(status) => status.index, + other => panic!("expected a status, got {other:?}"), + } +} + +fn until_index(client: &mut Client, what: &str, predicate: impl Fn(&IndexStatus) -> bool) { + for at in 0..2000 { + if predicate(&index_status(client, &format!("status-{at}"))) { + return; + } + std::thread::sleep(Duration::from_millis(5)); + } + panic!("the index never reported {what}"); +} + +fn healthy() -> Harness { + let (embedder, index) = (FakeEmbedder::new("d1"), FakeIndex::default()); + Harness::start(shelf(), embedder, index, Duration::from_secs(30)) +} + +/// A semantic answer is the mind's: the index names a resolved question, a +/// withdrawn resolution, a superseded ruling and a document the mind does not +/// hold, all scoring above the two that stand, and only the two that stand come +/// back, in score order. The selection's other predicates still cut the +/// candidates, and the embedder saw the query behind its instruction while the +/// index was asked for an oversample. +#[test] +fn a_semantic_query_returns_only_documents_the_mind_holds_and_the_selection_admits() { + let harness = healthy(); + let mut client = harness.client(); + until_index(&mut client, "Current", |status| *status == IndexStatus::Current); + harness.index.state.lock().unwrap().hits = vec![ + hit("question", id("question", "Q1"), 0.95), + hit("resolution", id("resolution", "question.Q2.n1"), 0.9), + hit("ruling", id("ruling", "R3"), 0.85), + hit("question", id("question", "Q99"), 0.8), + hit("ruling", id("ruling", "R4"), 0.5), + hit("question", id("question", "Q2"), 0.4), + ]; + + let standing_only = ask(&mut client, "m-1", &semantic(standing(), 5)); + assert_eq!(ids(&standing_only), vec![id("ruling", "R4"), id("question", "Q2")]); + let HuginnMindResponse::Query(page) = &standing_only else { unreachable!() }; + assert_eq!((page.matched, page.next.as_ref()), (2, None)); + + let rulings = Selection { schemas: Some(vec![PipelineKind::Ruling.type_id().into()]), ..standing() }; + assert_eq!(ids(&ask(&mut client, "m-2", &semantic(rulings, 5))), vec![id("ruling", "R4")], "the selection's schemas still apply"); + + let everything = ids(&ask(&mut client, "m-3", &semantic(Selection::default(), 5))); + assert!(everything.contains(&id("question", "Q1")) && !everything.contains(&id("question", "Q99")), + "a hit the selection does not exclude is shown, and one the mind does not hold never is: {everything:?}"); + + let embedded = harness.embedder.texts(); + assert!(embedded.iter().any(|text| text.starts_with("Instruct: ") && text.ends_with("\nQuery: who owns the state")), "{embedded:?}"); + let searches = harness.index.state.lock().unwrap().searches.clone(); + assert!(searches.iter().all(|(name, _, limit)| *name == collection_name(&slug(INSTANCE)) && *limit == 20), "{searches:?}"); +} + +/// A cursor cannot resume a ranked answer: it is refused by name, and nothing +/// is embedded or searched for it. +#[test] +fn a_cursor_with_a_semantic_query_is_refused_and_asks_the_index_nothing() { + let harness = healthy(); + let mut client = harness.client(); + until_index(&mut client, "Current", |status| *status == IndexStatus::Current); + let embedded = harness.embedder.texts().len(); + + let with_cursor = Selection { cursor: Some("anything".into()), ..standing() }; + let refusal = ask(&mut client, "m-1", &semantic(with_cursor, 5)); + assert!( + matches!(&refusal, HuginnMindResponse::Refused(MindRefusal::SelectionInvalid { field, .. }) if field == "cursor"), + "{refusal:?}" + ); + assert_eq!(harness.embedder.texts().len(), embedded); + assert!(harness.index.state.lock().unwrap().searches.is_empty()); +} + +/// The loop never waits on the network. The embedder is shut, the worker is +/// inside it with one client's semantic query, and a second client's `whoami` +/// and plain query are answered at once; the first is answered when the gate +/// opens. The gate counts arrivals, so this observes where the work is and not +/// how long it took. +#[test] +fn another_session_is_answered_while_a_semantic_search_is_outstanding() { + let gate = Gate::shut(); + gate.open(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let harness = Harness::start(shelf(), embedder, FakeIndex::default(), Duration::from_secs(30)); + let mut asker = harness.client(); + let mut second = harness.client(); + let mut other = harness.client(); + until_index(&mut other, "Current", |status| *status == IndexStatus::Current); + let held = documents(&mut other, "m-count"); + harness.index.state.lock().unwrap().hits = vec![hit("ruling", id("ruling", "R4"), 0.5)]; + + gate.shut_again(); + send(&mut asker, "m-ask", &semantic(standing(), 5)); + gate.wait_arrived(1); + assert!(receive(&mut asker, "m-ask", 20).is_none(), "the question is not answered while its embedding is outstanding"); + // A second question waits behind the first, and each is answered to its own asker. + send(&mut second, "m-ask-2", &semantic(Selection { schemas: Some(vec![PipelineKind::Question.type_id().into()]), ..Selection::default() }, 5)); + + assert!(matches!(ask(&mut other, "m-who", &HuginnMindRequest::Whoami), HuginnMindResponse::Whoami(_))); + assert_eq!(documents(&mut other, "m-plain"), held, "a plain query is answered too"); + + gate.open(); + let answered = receive(&mut asker, "m-ask", 3000).expect("the search is answered once the embedder returns"); + assert_eq!(ids(&answered), vec![id("ruling", "R4")]); + let second_answer = receive(&mut second, "m-ask-2", 3000).expect("the second search is answered too, on its own session"); + assert_eq!(ids(&second_answer), Vec::::new(), "the index offered a ruling, which its selection of questions does not admit"); +} + +/// A search the index never finishes is answered at the deadline the client +/// was promised, typed and naming the wait, never as an empty page. +#[test] +fn a_search_the_index_never_finishes_is_answered_unavailable_at_the_deadline() { + let gate = Gate::shut(); + gate.open(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let harness = Harness::start(shelf(), embedder, FakeIndex::default(), Duration::from_millis(150)); + let mut client = harness.client(); + until_index(&mut client, "Current", |status| *status == IndexStatus::Current); + + gate.shut_again(); + let answered = ask(&mut client, "m-late", &semantic(standing(), 5)); + let HuginnMindResponse::Refused(MindRefusal::Unavailable { detail }) = &answered else { panic!("{answered:?}") }; + assert!(detail.contains("did not finish"), "{detail}"); + gate.open(); + assert!(matches!(ask(&mut client, "m-after", &HuginnMindRequest::Whoami), HuginnMindResponse::Whoami(_)), "the late answer is dropped, not misdelivered"); +} + +/// An index that cannot answer is a typed `Unavailable` and never an empty +/// page, in each way it cannot: the embedder is down when the query arrives; +/// the worker is failing; the collection belongs to someone else. Plain +/// queries are unaffected throughout. +#[test] +fn an_index_that_cannot_answer_is_unavailable_and_plain_queries_still_work() { + let unavailable = |response: HuginnMindResponse| match response { + HuginnMindResponse::Refused(MindRefusal::Unavailable { detail }) => detail, + other => panic!("expected Unavailable, got {other:?}"), + }; + + // The embedder goes down after the index was Current: the query itself fails. + let harness = healthy(); + let mut client = harness.client(); + until_index(&mut client, "Current", |status| *status == IndexStatus::Current); + let held = documents(&mut client, "m-count"); + harness.embedder.state.lock().unwrap().down = true; + let detail = unavailable(ask(&mut client, "m-down", &semantic(standing(), 5))); + assert!(detail.contains("the embedder is down"), "{detail}"); + assert_eq!(documents(&mut client, "m-plain"), held); + + // A write while it is down makes the worker Failing, and the next query is + // refused without being asked of it. + let admit = HuginnMindRequest::Admit(batch(INSTANCE, vec![question("Q7")])); + assert!(matches!(ask(&mut client, "m-admit", &admit), HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. }))); + until_index(&mut client, "Failing", |status| matches!(status, IndexStatus::Failing { .. })); + let searches = harness.index.state.lock().unwrap().searches.len(); + let detail = unavailable(ask(&mut client, "m-failing", &semantic(standing(), 5))); + assert!(detail.contains("failing") && detail.contains("the embedder is down"), "{detail}"); + assert_eq!(harness.index.state.lock().unwrap().searches.len(), searches); + assert_eq!(documents(&mut client, "m-plain-2"), held + 1, "the admitted question is in the mind whatever the index says"); + + // A collection that is another writer's is Refused, and so are queries. + let (embedder, index) = (FakeEmbedder::new("d1"), FakeIndex::default()); + index.plant(&collection_name(&slug(INSTANCE)), Described::Unlabelled); + let refused = Harness::start(shelf(), embedder, index, Duration::from_secs(30)); + let mut client = refused.client(); + until_index(&mut client, "Refused", |status| matches!(status, IndexStatus::Refused { .. })); + let detail = unavailable(ask(&mut client, "m-refused", &semantic(standing(), 5))); + assert!(detail.contains("refused") && detail.contains("no Huginn metadata"), "{detail}"); + assert_eq!(documents(&mut client, "m-plain-3"), held); +} + +/// The ids a daemon reports as unheld are exactly the hits its mind has no +/// document for: an absent id, and an id under a kind it is not. +#[test] +fn a_hit_the_mind_holds_no_document_for_is_named_unheld() { + let root = tempfile::tempdir().unwrap(); + let mut daemon = crate::daemon::tests::open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); + let identity = crate::daemon::tests::identity(INSTANCE); + let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = + daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity])), now()).answered() + else { + panic!("the identity was not admitted") + }; + let held = writes[0].clone(); + let absent = PipelineRef { kind: held.kind, id: Short(id("instance", "nobody")) }; + let mis_kinded = PipelineRef { kind: PipelineKind::Question, id: held.id.clone() }; + let hits = vec![(held.clone(), 0.9), (absent.clone(), 0.8), (mis_kinded.clone(), 0.7)]; + assert_eq!(daemon.unheld(&hits), vec![absent.id.0.as_str(), mis_kinded.id.0.as_str()]); + assert!(daemon.unheld(&vec![(held, 0.9)]).is_empty()); +} diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index 287f7b9..593ae63 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -56,7 +56,7 @@ use std::collections::BTreeMap; use std::net::{SocketAddr, UdpSocket}; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; +use std::time::{Duration, Instant}; use anyhow::{Context, Result, ensure}; use base64::Engine; @@ -64,7 +64,7 @@ use base64::engine::general_purpose::STANDARD; use chrono::{DateTime, Utc}; use cultnet_rs::{ CULTNET_OPERATION_CONNECTION_ID, CultNetMessage, CultNetRudpServerEvent, CultNetRudpServerHub, - CultNetRudpServerHubOptions, CultNetSchemaKind, CultNetSchemaRegistration, CultNetSchemaRegistry, + CultNetRudpServerHubOptions, CultNetRudpServerSessionContext, CultNetSchemaKind, CultNetSchemaRegistration, CultNetSchemaRegistry, CultNetWireContract, answer_content_chunk_request, decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, pack_content, }; @@ -78,7 +78,7 @@ use huginn_mind::{Mind, MindRefusal, MindStore, OwnedRedbMessagePackBackingStore use crate::bodies::{ DEFERRED_BUDGET_BYTES, DEFERRED_CHUNK_BYTES, DeferredBodies, MAX_DEFERRED_BODY_BYTES, }; -use crate::daemon::{Daemon, IndexSink, runtime_id}; +use crate::daemon::{Daemon, Handled, IndexSink, Search, SearchTicket, runtime_id}; use crate::envelope::{OperationFailure, decode_request, encode_failure, encode_response}; use crate::index::{Backoff, Embedder, VectorIndex, WorkerSink}; @@ -88,6 +88,10 @@ pub struct ServeOptions { pub session_timeout: Duration, pub idle_sleep: Duration, pub deferred_ttl: Duration, + /// How long a semantic query may wait for the index before it is answered + /// `Unavailable`. The worker's own calls time out on their own; this is + /// what the client is promised. + pub search_timeout: Duration, } impl Default for ServeOptions { @@ -96,6 +100,7 @@ impl Default for ServeOptions { session_timeout: Duration::from_secs(30), idle_sleep: Duration::from_millis(2), deferred_ttl: Duration::from_secs(60), + search_timeout: Duration::from_secs(30), } } } @@ -250,33 +255,54 @@ fn overlong_echo(message: &CultNetMessage) -> Option<&'static str> { echoed.into_iter().find(|(_, value)| value.len() > MAX_ECHOED_FIELD_BYTES).map(|(name, _)| name) } -/// One message in, one message out. Nothing here reaches a mind except through -/// `Daemon::handle`, and an envelope that does not decode never gets that far. -/// A chunk request is answered from `bodies` alone, on the session it came in -/// on; every operation's reply leaves through `deliver`. +/// What one message came to: a message to send back now, or a semantic query +/// the index has yet to answer, which the loop holds against the session it +/// came in on. +pub enum Routed { + Reply(CultNetMessage), + Search { message_id: String, operation: &'static str, search: Search }, +} + +impl Routed { + /// The reply, for a message that cannot be a search. + #[cfg(test)] + pub(crate) fn reply(self) -> CultNetMessage { + match self { + Self::Reply(reply) => reply, + Self::Search { .. } => panic!("expected a reply, got a search"), + } + } +} + +/// One message in, one message out, or a search to wait for. Nothing here +/// reaches a mind except through `Daemon::handle`, and an envelope that does +/// not decode never gets that far. A chunk request is answered from `bodies` +/// alone, on the session it came in on; every operation's reply leaves through +/// `respond`. pub fn answer>( daemon: &mut Daemon, registry: &CultNetSchemaRegistry, bodies: &mut DeferredBodies, message: CultNetMessage, now: DateTime, -) -> CultNetMessage { +) -> Routed { let runtime_id = daemon.runtime_id(); bodies.expire(now); if let Some(field) = overlong_echo(&message) { - return CultNetMessage::Error { + return Routed::Reply(CultNetMessage::Error { error: format!("{MIND_SERVICE_ID} refuses a {field} over {MAX_ECHOED_FIELD_BYTES} bytes"), code: None, details: None, - }; + }); } - match &message { + Routed::Reply(match &message { CultNetMessage::OperationRequest { message_id, operation, .. } => match decode_request(&message) { Ok((message_id, request)) => { let operation = request.operation(); - let response = daemon.handle(request, now); - let reply = encode_or_fail(&message_id, operation, &response, &runtime_id); - deliver(reply, &message_id, operation, &runtime_id, bodies, now) + match daemon.handle(request, now) { + Handled::Answered(response) => respond(&response, &message_id, operation, &runtime_id, bodies, now), + Handled::Searching(search) => return Routed::Search { message_id, operation, search }, + } } Err(failure) => encode_failure(message_id, operation, &failure, &runtime_id), }, @@ -295,7 +321,21 @@ pub fn answer>( code: None, details: None, }, - } + }) +} + +/// One answer as the message that carries it: encoded, then delivered whole, +/// deferred or refused by `deliver`. +fn respond( + response: &HuginnMindResponse, + message_id: &str, + operation: &str, + runtime_id: &str, + bodies: &mut DeferredBodies, + now: DateTime, +) -> CultNetMessage { + let reply = encode_or_fail(message_id, operation, response, runtime_id); + deliver(reply, message_id, operation, runtime_id, bodies, now) } /// One response in its envelope, or the failure that says it did not encode. @@ -397,10 +437,24 @@ fn deliver( deferred } +/// A semantic query on its way through the index: who asked, what to call the +/// answer, the search to finish, and when the client stops being promised one. +struct Waiting { + session: CultNetRudpServerSessionContext, + message_id: String, + operation: &'static str, + search: Search, + until: Instant, +} + /// Until `stopping`: expire what timed out, resend what was not acknowledged, -/// then answer every frame waiting on the socket. A hostile datagram and a -/// departed session are logged and served past, never fatal. This is the -/// crate's only clock read, once per frame. +/// answer every frame waiting on the socket, then answer every semantic query +/// the index has finished or that has waited out `search_timeout`. A semantic +/// query is never answered inline: `answer` hands back a `Search`, the loop +/// holds it beside its session and goes on serving, and the embedder and the +/// vector store are the worker's to wait on. A hostile datagram and a departed +/// session are logged and served past, never fatal. The clock is read once per +/// frame, and once per pass for the searches' deadlines. pub fn run>( daemon: &mut Daemon, hub: &mut CultNetRudpServerHub, @@ -410,6 +464,7 @@ pub fn run>( ) -> Result<()> { let timeout_ms = options.session_timeout.as_millis() as u64; let mut bodies = DeferredBodies::new(DEFERRED_BUDGET_BYTES, options.deferred_ttl); + let mut waiting: BTreeMap = BTreeMap::new(); while !stopping.load(Ordering::Relaxed) { hub.remove_timed_out_sessions(timeout_ms); if let Err(error) = hub.poll_resends() { @@ -439,19 +494,54 @@ pub fn run>( continue; } }; - let reply = answer(daemon, registry, &mut bodies, message, Utc::now()); - if let Err(error) = hub.send_schema_message(&session, &reply) { - eprintln!("huginn: {} did not receive its reply: {error:#}", session.remote_addr); + match answer(daemon, registry, &mut bodies, message, Utc::now()) { + Routed::Reply(reply) => send(hub, &session, &reply), + Routed::Search { message_id, operation, search } => { + let until = Instant::now() + options.search_timeout; + waiting.insert(search.ticket, Waiting { session, message_id, operation, search, until }); + } } served += 1; } - if served == 0 { + let finished = collect_searches(daemon, &waiting, options); + let answered = !finished.is_empty(); + for (ticket, found) in finished { + let Some(asked) = waiting.remove(&ticket) else { continue }; + let response = daemon.finish(asked.search, found); + let reply = respond(&response, &asked.message_id, asked.operation, &daemon.runtime_id(), &mut bodies, Utc::now()); + send(hub, &asked.session, &reply); + } + if served == 0 && !answered { std::thread::sleep(options.idle_sleep); } } Ok(()) } +/// What the index has finished, then what has waited too long: the latter as +/// an `Unavailable` detail. A search that finishes after its deadline finds +/// nothing waiting for it and is dropped. +fn collect_searches>( + daemon: &mut Daemon, + waiting: &BTreeMap, + options: &ServeOptions, +) -> Vec<(SearchTicket, Result)> { + let mut collected = daemon.searched(); + let now = Instant::now(); + for (ticket, asked) in waiting { + if asked.until <= now { + collected.push((*ticket, Err(format!("the semantic search did not finish within {:?}", options.search_timeout)))); + } + } + collected +} + +fn send(hub: &mut CultNetRudpServerHub, session: &CultNetRudpServerSessionContext, reply: &CultNetMessage) { + if let Err(error) = hub.send_schema_message(session, reply) { + eprintln!("huginn: {} did not receive its reply: {error:#}", session.remote_addr); + } +} + #[cfg(test)] mod tests { use super::*; @@ -472,7 +562,7 @@ mod tests { use std::sync::atomic::AtomicBool; fn documents(daemon: &mut Daemon) -> u32 { - match daemon.handle(HuginnMindRequest::Whoami, now()) { + match daemon.handle(HuginnMindRequest::Whoami, now()).answered() { HuginnMindResponse::Whoami(MindStatus { documents, .. }) => documents, other => panic!("expected a status, got {other:?}"), } @@ -539,14 +629,14 @@ mod tests { ("m-5b", request("m-5b", MIND_SERVICE_ID, "Admit", MIND_REQUEST_SCHEMA, admit), "operation-mismatch"), ]; for (id, message, code) in cases { - let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), message, now()); + let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), message, now()).reply(); assert_eq!(rejected(&reply).0, id, "the client's own correlation key is echoed"); assert_eq!(rejected(&reply).1, code); } // The sixth code, `not-an-operation-request`, is the codec's and is // pinned in `envelope::tests`: `answer` never produces it, because a // message of another family is answered with `Error` instead. - let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), CultNetMessage::Error { error: "hello".into(), code: None, details: None }, now()); + let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), CultNetMessage::Error { error: "hello".into(), code: None, details: None }, now()).reply(); let CultNetMessage::Error { error, .. } = &reply else { panic!("expected an error, got {reply:?}") }; assert!( !error.is_empty() && error.contains("cultnet.operation_request.v0") && error.contains("cultnet.schema_catalog_request.v0"), @@ -558,7 +648,7 @@ mod tests { // A mind's refusal is an answer on the response schema, not a failure // of the envelope: the client decodes it as the typed refusal it is. let read = HuginnMindRequest::Query { instance: slug(OTHER), selection: Selection::default(), semantic: None }; - let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), encode_request("m-r", &read, None).unwrap(), now()); + let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), encode_request("m-r", &read, None).unwrap(), now()).reply(); let CultNetMessage::OperationResponse { status, payload_schema, .. } = &reply else { panic!("expected an operation response, got {reply:?}"); }; @@ -576,7 +666,7 @@ mod tests { schema_ids: None, kinds: None, }; - let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), catalog, now()); + let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), catalog, now()).reply(); let CultNetMessage::SchemaCatalogResponse { message_id, schemas } = reply else { panic!("expected a catalog response"); }; @@ -749,7 +839,7 @@ mod tests { at: DateTime, ) -> CultMeshCdnArtifactManifest { let registry = schema_registry().unwrap(); - deferred_manifest(&answer(daemon, ®istry, bodies, encode_request(message_id, read, None).unwrap(), at)) + deferred_manifest(&answer(daemon, ®istry, bodies, encode_request(message_id, read, None).unwrap(), at).reply()) } /// Whether the first chunk of a manifest is still served at an injected time. @@ -760,7 +850,7 @@ mod tests { at: DateTime, ) -> bool { let registry = schema_registry().unwrap(); - found(&answer(daemon, ®istry, bodies, chunk_request(manifest, 0), at)) + found(&answer(daemon, ®istry, bodies, chunk_request(manifest, 0), at).reply()) } /// An answer larger than one send can carry is deferred to the body plane @@ -780,7 +870,7 @@ mod tests { let registry = schema_registry().unwrap(); let sized = |daemon: &mut BodyDaemon, id: PipelineRef| { - let view = daemon.handle(HuginnMindRequest::View { instance: slug(INSTANCE), id }, now()); + let view = daemon.handle(HuginnMindRequest::View { instance: slug(INSTANCE), id }, now()).answered(); let message = encode_response("m-0", "view", &view, "huginn-yggdrasil").unwrap(); encoded_len(&message) }; @@ -800,7 +890,7 @@ mod tests { ("m-v", HuginnMindRequest::View { instance: slug(INSTANCE), id: wide }), ]; let expected: Vec = - reads.iter().map(|(_, read)| daemon.handle(read.clone(), now())).collect(); + reads.iter().map(|(_, read)| daemon.handle(read.clone(), now()).answered()).collect(); let mut hub = bind("127.0.0.1:0".parse().unwrap(), &daemon.runtime_id()).unwrap(); let endpoint = format!("rudp://{}", hub.local_addr().unwrap()); @@ -1065,7 +1155,7 @@ mod tests { let registry = schema_registry().unwrap(); let wide = spec_ref(&mut daemon, WIDE_CUT); let read = HuginnMindRequest::View { instance: slug(INSTANCE), id: wide }; - let expected = rmp_serde::to_vec_named(&daemon.handle(read.clone(), now())).unwrap(); + let expected = rmp_serde::to_vec_named(&daemon.handle(read.clone(), now()).answered()).unwrap(); let t0 = now(); let after = |seconds: i64| t0 + chrono::Duration::seconds(seconds); let mut bodies = fresh_bodies(); @@ -1073,7 +1163,7 @@ mod tests { let manifest = ask_deferred(&mut daemon, &mut bodies, "m-v", &read, t0); assert!(manifest.chunks.len() >= 2, "the fixture needs a second chunk to serve"); let mut chunk = |index: usize, at: i64| { - answer(&mut daemon, ®istry, &mut bodies, chunk_request(&manifest, index), after(at)) + answer(&mut daemon, ®istry, &mut bodies, chunk_request(&manifest, index), after(at)).reply() }; assert!(found(&chunk(0, 59)), "inside the TTL"); assert!(found(&chunk(1, 118)), "serving a chunk slid the expiry past the first TTL"); @@ -1086,7 +1176,7 @@ mod tests { let again = ask_deferred(&mut daemon, &mut bodies, "m-v2", &read, after(179)); assert_eq!(again.content_hash, manifest.content_hash, "the same answer is the same body"); let bytes = fetch_content(&again, MAX_DEFERRED_BODY_BYTES, |request| { - Ok(answer(&mut daemon, ®istry, &mut bodies, request, after(180))) + Ok(answer(&mut daemon, ®istry, &mut bodies, request, after(180)).reply()) }) .unwrap(); assert_eq!(bytes, expected); @@ -1102,7 +1192,7 @@ mod tests { let view = HuginnMindRequest::View { instance: slug(INSTANCE), id: wide }; let query = document_query(); let size = |daemon: &mut BodyDaemon, read: &HuginnMindRequest| { - rmp_serde::to_vec_named(&daemon.handle(read.clone(), now())).unwrap().len() as u64 + rmp_serde::to_vec_named(&daemon.handle(read.clone(), now()).answered()).unwrap().len() as u64 }; let (view_bytes, query_bytes) = (size(&mut daemon, &view), size(&mut daemon, &query)); // Room for the larger body alone, and not for both. @@ -1186,7 +1276,7 @@ mod tests { ("record_key", chunk("m", "abc", &huge)), ]; for (case, message) in refused { - let reply = answer(&mut daemon, ®istry, &mut bodies, message, now()); + let reply = answer(&mut daemon, ®istry, &mut bodies, message, now()).reply(); let CultNetMessage::Error { error, .. } = &reply else { panic!("{case}: expected a transport error, got {reply:?}"); }; @@ -1195,9 +1285,9 @@ mod tests { } assert_eq!(documents(&mut daemon), 0); - let reply = answer(&mut daemon, ®istry, &mut bodies, whoami(&at, MIND_SERVICE_ID, "whoami", MIND_REQUEST_SCHEMA), now()); + let reply = answer(&mut daemon, ®istry, &mut bodies, whoami(&at, MIND_SERVICE_ID, "whoami", MIND_REQUEST_SCHEMA), now()).reply(); assert_eq!(decode_response(&reply).unwrap().0, at, "a field of exactly the bound is served"); - let reply = answer(&mut daemon, ®istry, &mut bodies, chunk(&at, &at, &at), now()); + let reply = answer(&mut daemon, ®istry, &mut bodies, chunk(&at, &at, &at), now()).reply(); assert!(!found(&reply), "an unknown chunk is answered found: false, not refused at the door"); } @@ -1233,7 +1323,7 @@ mod tests { ..Selection::default() }; let HuginnMindResponse::Query(page) = - daemon.handle(HuginnMindRequest::Query { instance: slug(INSTANCE), selection, semantic: None }, now()) + daemon.handle(HuginnMindRequest::Query { instance: slug(INSTANCE), selection, semantic: None }, now()).answered() else { panic!("expected a page"); }; diff --git a/crates/huginn-daemon/tests/live_index_smoke.rs b/crates/huginn-daemon/tests/live_index_smoke.rs index 8baaf69..dbb6f49 100644 --- a/crates/huginn-daemon/tests/live_index_smoke.rs +++ b/crates/huginn-daemon/tests/live_index_smoke.rs @@ -11,7 +11,7 @@ use std::collections::BTreeSet; -use huginn_daemon::index::ollama::OllamaEmbedder; +use huginn_daemon::index::ollama::{OllamaEmbedder, QUERY_INSTRUCTION}; use huginn_daemon::index::qdrant::QdrantIndex; use huginn_daemon::index::{ CollectionMeta, Described, Embedder, MANAGED_BY, Point, PointPayload, VectorIndex, point_id, @@ -90,7 +90,17 @@ fn the_adapters_embed_write_query_and_clean_up_a_scratch_collection() { let (status, answer) = raw("POST", &format!("{qdrant}/collections/{name}/points/search"), Some(&query)); assert_eq!(status, 200, "{answer}"); assert!(answer.contains(&points[0].id), "{answer}"); - println!("{name}: described, created, upserted {} points, queried", points.len()); + // The read side: a paraphrase that shares no exact words with the + // first document finds it, through the adapter, with the query + // instruction the worker uses. + let asked = format!("Instruct: {QUERY_INSTRUCTION} +Query: Who is responsible for keeping the memory?"); + let query = embedder.embed(&[asked]).expect("the query embeds").remove(0); + let hits = store.search(name, &query, 2).unwrap(); + assert_eq!(hits.len(), 2, "{hits:?}"); + assert_eq!((hits[0].doc_id.as_str(), hits[0].kind.as_str()), ("smoke:doc:0", "question"), "{hits:?}"); + assert!(hits[0].score > hits[1].score, "{hits:?}"); + println!("{name}: described, created, upserted {} points, queried, searched: {hits:?}", points.len()); } })); for name in [&plain, &dotted] { diff --git a/crates/huginn-mind/src/query.rs b/crates/huginn-mind/src/query.rs index a85e353..a3d377b 100644 --- a/crates/huginn-mind/src/query.rs +++ b/crates/huginn-mind/src/query.rs @@ -16,12 +16,15 @@ //! over the documents admitted at or before the page's `asOf`), the //! projection to a header or a whole view, and the typing of the edges. //! -//! `semantic` is accepted by the type and refused typed until Cut 11b wires the -//! read side of the index. No clock, no store handle, no network. +//! `rank` is the semantic read: the daemon's index supplies candidate ids and +//! scores, and this module joins them back through the mind and lets the +//! selection decide. No clock, no store handle, no network. use std::collections::BTreeMap; -use cultnet_rs::{Cursor, PROJECTION_DOCUMENT, Selection, select, validate}; +use cultnet_rs::{ + Cursor, EdgeAnchor, EdgeMatch, Evaluation, LIMIT_MAX, LIMIT_MIN, PROJECTION_DOCUMENT, Row, Selection, select, validate, +}; use epiphany_pipeline::{ ClaimOutcome, CommitRange, Date, FindingConfidence, FindingOrigin, FindingSeverity, Label, Line, OrgRepo, PipelineDocument, PipelineKind, PipelineRef, PipelineResolution, ResolutionOutcome, RulingAuthority, Sha, Short, @@ -72,8 +75,8 @@ pub struct PipelineDocumentView { pub status: PipelineStatus, } -/// A semantic query, accepted by the type and refused until Cut 11 owns the -/// index. Cut 11 also owns which fields of each kind are text, so no substring +/// A semantic query: the text to find documents near, and how many. The index +/// owns which fields of each kind are text (`index_text`), so no substring /// filter lives here to become a second answer to that question. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct SemanticQuery { @@ -425,22 +428,12 @@ impl Mind { /// One typed selection over the mind, as of one snapshot. /// - /// In order: `semantic` is refused before any read; the substrate checks - /// the selection's names; the organ's door checks its values; the - /// snapshot is the cursor's own `asOf` when there is a cursor and the - /// head otherwise (a cursor from beyond the head is one this mind could - /// not have minted); the substrate evaluates; the page is projected. The - /// evaluator's answer is never re-filtered here. - pub fn query( - &self, - selection: &Selection, - semantic: Option<&SemanticQuery>, - ) -> Result { - if semantic.is_some() { - return Err(MindRefusal::Unavailable { - detail: "semantic query: the index is not wired (Cut 11)".into(), - }); - } + /// In order: the substrate checks the selection's names; the organ's door + /// checks its values; the snapshot is the cursor's own `asOf` when there + /// is a cursor and the head otherwise (a cursor from beyond the head is + /// one this mind could not have minted); the substrate evaluates; the page + /// is projected. The evaluator's answer is never re-filtered here. + pub fn query(&self, selection: &Selection) -> Result { validate(selection, &Vocabulary).map_err(cultnet_rs::SelectionRefusal::Invalid)?; let selection = refuse_values(selection)?; let head = receipt::head(self)?; @@ -458,6 +451,118 @@ impl Mind { }; let rows = Reader::at(self, as_of)?.rows()?; let evaluation = select(&Vocabulary, &rows, &selection, as_of, self.cursor_key())?; + Self::page(&selection, evaluation, as_of) + } + + /// What a semantic query must satisfy before anything is embedded: the + /// selection passes the same two doors `query` does, carries no cursor + /// (a ranked answer is not pageable), and asks for between one and + /// `LIMIT_MAX` hits over text that says something. Returns the selection + /// as the door leaves it. + fn semantic_selection(selection: &Selection, semantic: &SemanticQuery) -> Result { + let refused = |field: &str, value: Option, message: &str| MindRefusal::SelectionInvalid { + field: field.into(), + value, + message: message.into(), + }; + if selection.cursor.as_deref().is_some_and(|cursor| !cursor.is_empty()) { + return Err(refused("cursor", None, "a semantic answer is ranked, not paged: it carries no cursor")); + } + if semantic.text.0.trim().is_empty() { + return Err(refused("semantic.text", None, "a semantic query needs text")); + } + if !(LIMIT_MIN..=LIMIT_MAX).contains(&semantic.top_k) { + return Err(refused( + "semantic.top_k", + Some(semantic.top_k.to_string()), + &format!("top_k is between {LIMIT_MIN} and {LIMIT_MAX}"), + )); + } + validate(selection, &Vocabulary).map_err(cultnet_rs::SelectionRefusal::Invalid)?; + refuse_values(selection) + } + + /// Whether a semantic query may be started, decided before the index is + /// asked anything. `rank` runs the same check again over the same inputs. + pub fn check_semantic(&self, selection: &Selection, semantic: &SemanticQuery) -> Result<(), MindRefusal> { + Self::semantic_selection(selection, semantic).map(drop) + } + + /// The page a semantic query answers, from the candidates the index gave. + /// + /// The index supplies ids and scores and nothing else. A hit is a + /// candidate: it is dropped unless the mind holds a document of that kind + /// and id at the head, and what remains is decided by the selection, run + /// by the substrate over every row the mind has, its `keys` narrowed to + /// the candidates. Narrowing `keys` rather than the rows keeps the + /// evaluator's universe whole, so `cited` and `in_force` mean what they + /// mean in `query`. Nothing about a document's standing is read from the + /// index; `in_force` is the caller's to ask for and the mind's to derive. + /// + /// The order is the one thing the substrate cannot know: score, highest + /// first, ties by ordinal. The page holds at most the selection's limit + /// and at most `top_k` documents, `matched` counts every candidate the + /// selection passed, and `next` is never set. Only the best + /// `LIMIT_MAX` candidates are considered. + pub fn rank( + &self, + selection: &Selection, + semantic: &SemanticQuery, + hits: &[(PipelineRef, f32)], + ) -> Result { + let selection = Self::semantic_selection(selection, semantic)?; + let head = receipt::head(self)?; + let rows = Reader::at(self, head)?.rows()?; + + let mut best: Vec<&(PipelineRef, f32)> = hits.iter().collect(); + best.sort_by(|a, b| b.1.total_cmp(&a.1)); + best.truncate(LIMIT_MAX as usize); + let mut scores: BTreeMap<&str, f32> = BTreeMap::new(); + for row in &rows { + if let Some((_, score)) = best.iter().find(|(hit, _)| hit.kind == row.kind && hit.id.0 == row.key) { + scores.insert(row.key.as_str(), *score); + } + } + let mut keys: Vec = scores.keys().map(|key| key.to_string()).collect(); + if let Some(allowed) = &selection.keys { + keys.retain(|key| allowed.contains(key)); + } + let limit = selection.limit.unwrap_or(LIMIT_MAX).clamp(LIMIT_MIN, LIMIT_MAX).min(semantic.top_k) as usize; + + let mut evaluation = if keys.is_empty() { + Evaluation { rows: Vec::new(), matched: 0, edges: Vec::new(), next_cursor: None } + } else { + let narrowed = Selection { keys: Some(keys), limit: Some(LIMIT_MAX), cursor: None, ..selection.clone() }; + select(&Vocabulary, &rows, &narrowed, head, self.cursor_key())? + }; + let score = |row: &SelectionRow| scores.get(row.key.as_str()).copied().unwrap_or(f32::NEG_INFINITY); + evaluation.rows.sort_by(|a, b| score(b).total_cmp(&score(a)).then(a.ordinal().cmp(&b.ordinal()))); + evaluation.rows.truncate(limit); + evaluation.next_cursor = None; + let position = + |kind: PipelineKind, key: &str| evaluation.rows.iter().position(|row| row.kind == kind && row.key == key); + let mut kept: Vec<(usize, EdgeMatch)> = std::mem::take(&mut evaluation.edges) + .into_iter() + .filter_map(|edge| { + let owner = match edge.anchor { + EdgeAnchor::Citee => &edge.to, + EdgeAnchor::Citer => &edge.from, + }; + position(owner.kind, &owner.key).map(|at| (at, edge)) + }) + .collect(); + kept.sort_by_key(|(at, _)| *at); + evaluation.edges = kept.into_iter().map(|(_, edge)| edge).collect(); + Self::page(&selection, evaluation, head) + } + + /// The projection of an evaluation: a header or a whole view per row, and + /// the typed edges a hop traversed. + fn page( + selection: &Selection, + evaluation: Evaluation, + as_of: u64, + ) -> Result { let items = if selection.projection == PROJECTION_DOCUMENT { PipelinePageItems::Documents(evaluation.rows.iter().map(|row| row.view.clone()).collect()) } else { @@ -534,7 +639,7 @@ mod tests { /// The refusal a selection earns from a mind that would otherwise answer. fn refused(mind: &Mind, selection: &Selection) -> MindRefusal { - mind.query(selection, None).expect_err("the selection is refused") + mind.query(selection).expect_err("the selection is refused") } fn invalid(field: &str, value: &str) -> impl Fn(&MindRefusal) -> bool { @@ -687,7 +792,7 @@ mod tests { let refuses = |store: MemoryStore, label: &str| { let broken_mind = opened(store, INSTANCE); assert!( - matches!(broken_mind.query(&Selection::default(), None), Err(MindRefusal::Unavailable { .. })), + matches!(broken_mind.query(&Selection::default()), Err(MindRefusal::Unavailable { .. })), "{label}: query must refuse an undense chain" ); assert!( @@ -743,7 +848,7 @@ mod tests { orphaned.view(&r(K::Question, &orphan)).err(), Some(MindRefusal::Unavailable { detail: detail.clone() }) ); - assert_eq!(orphaned.query(&Selection::default(), None).err(), Some(MindRefusal::Unavailable { detail })); + assert_eq!(orphaned.query(&Selection::default()).err(), Some(MindRefusal::Unavailable { detail })); // The other half of the same rule: A10 admits one write per identity, // so two receipts naming one document is a store this organ cannot @@ -763,7 +868,7 @@ mod tests { .unwrap(); doubled.plant(schema_cache().unwrap().prepare_entry_named(&forged.receipt_id, &forged).unwrap().0); assert_eq!( - opened(doubled, INSTANCE).query(&Selection::default(), None).err(), + opened(doubled, INSTANCE).query(&Selection::default()).err(), Some(MindRefusal::Unavailable { detail: format!("document {}/{q1} is written by two receipts", K::Question.type_id()), }) @@ -795,11 +900,11 @@ mod tests { } opened(copy, INSTANCE) }; - let expected = mind.query(&Selection::default(), None).unwrap(); + let expected = mind.query(&Selection::default()).unwrap(); let constant = vec!["2000-01-01T00:00:00Z".to_string(); rows.len()]; let reversed = rows.iter().rev().map(|row| row.stored_at.clone()).collect::>(); for other in [reopened(constant), reopened(reversed)] { - assert_eq!(other.query(&Selection::default(), None).unwrap(), expected); + assert_eq!(other.query(&Selection::default()).unwrap(), expected); } } @@ -841,7 +946,7 @@ mod tests { } fn matching(mind: &Mind, selection: Selection) -> Vec { - let mut matched = ids(&mind.query(&selection, None).unwrap()); + let mut matched = ids(&mind.query(&selection).unwrap()); matched.sort(); matched } @@ -938,11 +1043,11 @@ mod tests { committed(admit(&mut mind, vec![question_n(9)])); committed(admit(&mut mind, vec![question_n(1)])); let selection = of_kinds(&[K::Question]); - assert_eq!(ids(&mind.query(&selection, None).unwrap()), vec![id("question", "Q9"), id("question", "Q1")]); + assert_eq!(ids(&mind.query(&selection).unwrap()), vec![id("question", "Q9"), id("question", "Q1")]); let newest_first = Selection { descending: true, ..selection }; - assert_eq!(ids(&mind.query(&newest_first, None).unwrap()), vec![id("question", "Q1"), id("question", "Q9")]); + assert_eq!(ids(&mind.query(&newest_first).unwrap()), vec![id("question", "Q1"), id("question", "Q9")]); // The facts on a header name the ordinal the order came from. - let page = mind.query(&of_kinds(&[K::Question]), None).unwrap(); + let page = mind.query(&of_kinds(&[K::Question])).unwrap(); let ordinals: Vec = headers(&page).iter().map(|header| header.admission.ordinal).collect(); assert_eq!(ordinals, vec![2, 3]); } @@ -959,13 +1064,13 @@ mod tests { } let questions = of_kinds(&[K::Question]); for (limit, expected) in [(None, 200), (Some(500), 200), (Some(0), 1), (Some(5), 5)] { - let page = mind.query(&Selection { limit, ..questions.clone() }, None).unwrap(); + let page = mind.query(&Selection { limit, ..questions.clone() }).unwrap(); assert_eq!(ids(&page).len(), expected, "limit {limit:?}"); assert_eq!(page.matched, 201, "the count is taken before the cap"); } - let first = mind.query(&questions, None).unwrap(); + let first = mind.query(&questions).unwrap(); assert_eq!(ids(&first)[0], id("question", "Q10"), "the earliest batch first, and inside it key order"); - let second = mind.query(&Selection { cursor: first.next.clone(), ..questions }, None).unwrap(); + let second = mind.query(&Selection { cursor: first.next.clone(), ..questions }).unwrap(); assert_eq!(ids(&second), vec![id("question", "Q201")], "the last batch's last key is the one the cap dropped"); assert_eq!((second.matched, second.next), (201, None)); assert_eq!(second.as_of, first.as_of); @@ -984,7 +1089,7 @@ mod tests { // Newest first, so page 1's last row is the head row: the boundary // sits at exactly `asOf`. let walk = Selection { descending: true, limit: Some(2), ..of_kinds(&[K::Question]) }; - let first = mind.query(&walk, None).unwrap(); + let first = mind.query(&walk).unwrap(); assert_eq!(ids(&first), vec![id("question", "Q3"), id("question", "Q2")]); assert_eq!((first.matched, first.as_of), (3, 4)); let cursor = first.next.clone().expect("Q1 is left"); @@ -992,16 +1097,16 @@ mod tests { let q1 = id("question", "Q1"); committed(admit(&mut mind, vec![answering("R1", &q1), question_n(4)])); - let second = mind.query(&Selection { cursor: Some(cursor.clone()), ..walk.clone() }, None).unwrap(); + let second = mind.query(&Selection { cursor: Some(cursor.clone()), ..walk.clone() }).unwrap(); assert_eq!(ids(&second), vec![q1.clone()]); assert_eq!(headers(&second)[0].status, PipelineStatusSummary::InForce, "the closing ruling landed after asOf"); assert_eq!((second.matched, second.as_of, second.next.as_deref()), (3, 4, None)); // A fresh walk starts at the new head and sees all of it. - let fresh = mind.query(&walk, None).unwrap(); + let fresh = mind.query(&walk).unwrap(); assert_eq!((fresh.matched, fresh.as_of), (4, 5)); assert_eq!(ids(&fresh), vec![id("question", "Q4"), id("question", "Q3")]); - let closed = mind.query(&Selection { cursor: fresh.next.clone(), ..walk.clone() }, None).unwrap(); + let closed = mind.query(&Selection { cursor: fresh.next.clone(), ..walk.clone() }).unwrap(); assert!(matches!(headers(&closed)[1].status, PipelineStatusSummary::Resolved { .. }), "Q1 is closed at head"); // A cursor is bound to its selection: change the order and it is not @@ -1023,10 +1128,10 @@ mod tests { cursor: Some(Cursor::mint(as_of, &rows[0], &selection, mind.cursor_key())), ..selection.clone() }; - assert!(mind.query(&at(head), None).is_ok(), "a cursor at exactly the head answers"); + assert!(mind.query(&at(head)).is_ok(), "a cursor at exactly the head answers"); assert!(matches!(refused(&mind, &at(head + 1)), MindRefusal::CursorInvalid { .. })); let stale = at(1); - assert!(mind.query(&stale, None).is_ok(), "an older snapshot is still answerable: the mind is append-only"); + assert!(mind.query(&stale).is_ok(), "an older snapshot is still answerable: the mind is append-only"); } /// The door refuses a value outside its alias's domain, naming the field @@ -1034,7 +1139,7 @@ mod tests { #[test] fn a_root_outside_the_grammar_is_refused_not_empty() { let mind = worked(); - assert!(mind.query(&with(Selection::default(), any_of("root", &[CAMPAIGN])), None).unwrap().matched > 0); + assert!(mind.query(&with(Selection::default(), any_of("root", &[CAMPAIGN]))).unwrap().matched > 0); // The bad value is the second in its list and the predicate is the // second in its selection: the door reads them all. let bad = with(with(Selection::default(), any_of("in_force", &["true"])), any_of("root", &[CAMPAIGN, "a/../b"])); @@ -1060,8 +1165,8 @@ mod tests { let mind = worked(); let severity = |value: &str| with(of_kinds(&[K::Finding]), any_of("severity", &[value])); assert!(invalid("fields[0].values", "high")(&refused(&mind, &severity("high")))); - assert_eq!(ids(&mind.query(&severity("High"), None).unwrap()), vec![id("finding", "cut-9.s1.F1")]); - assert!(mind.query(&severity("Low"), None).unwrap().matched == 0, "a value in the domain that matches nothing is empty"); + assert_eq!(ids(&mind.query(&severity("High")).unwrap()), vec![id("finding", "cut-9.s1.F1")]); + assert!(mind.query(&severity("Low")).unwrap().matched == 0, "a value in the domain that matches nothing is empty"); let in_force = with(Selection::default(), any_of("in_force", &["yes"])); assert!(invalid("fields[0].values", "yes")(&refused(&mind, &in_force))); } @@ -1077,7 +1182,7 @@ mod tests { for alias in ["root", "in_force", "faculty", "repo", "cut", "severity", "confidence", "origin", "authority", "claim_outcome", "outcome"] { for value in cultnet_rs::Row::values(row, alias) { let selection = with(Selection::default(), any_of(alias, &[&value])); - assert!(mind.query(&selection, None).unwrap().matched >= 1, "{alias}={value}"); + assert!(mind.query(&selection).unwrap().matched >= 1, "{alias}={value}"); } } } @@ -1097,7 +1202,7 @@ mod tests { } let q1 = id("question", "Q1"); let held = Selection { keys: Some(vec![q1.clone()]), ..Selection::default() }; - assert_eq!(ids(&mind.query(&held, None).unwrap()), vec![q1]); + assert_eq!(ids(&mind.query(&held).unwrap()), vec![q1]); } /// V24 at the selection's door: a `cites` target whose kind and key @@ -1109,7 +1214,7 @@ mod tests { let wrong = Selection { cites: Some(cites(K::Ruling, &q1, None)), ..Selection::default() }; assert!(invalid("cites.target", &q1)(&refused(&mind, &wrong))); let right = Selection { cites: Some(cites(K::Question, &q1, Some("source"))), ..Selection::default() }; - assert_eq!(ids(&mind.query(&right, None).unwrap()), vec![id("follow_up", "FU-1")]); + assert_eq!(ids(&mind.query(&right).unwrap()), vec![id("follow_up", "FU-1")]); } /// The substrate's own refusals cross as themselves: an undeclared alias @@ -1133,7 +1238,7 @@ mod tests { committed(admit(&mut mind, vec![question_n(1)])); let q1 = id("question", "Q1"); let one = Selection { keys: Some(vec![q1.clone()]), ..Selection::default() }; - let page = mind.query(&one, None).unwrap(); + let page = mind.query(&one).unwrap(); let expected = view(&mind, K::Question, &q1); assert_eq!(page.edges, None, "no hop, no edges"); assert_eq!(headers(&page), [PipelineDocumentSummary::of(&expected)]); @@ -1145,7 +1250,7 @@ mod tests { raised_in: None, asked_on: date(), }); - let whole = mind.query(&Selection { projection: "document".into(), ..one }, None).unwrap(); + let whole = mind.query(&Selection { projection: "document".into(), ..one }).unwrap(); assert_eq!(whole.items, PipelinePageItems::Documents(vec![expected])); } @@ -1221,7 +1326,7 @@ mod tests { for role in CitationRole::ALL { let selection = Selection { cited: Some(Incoming { role: role.name().into(), exists: true }), ..Selection::default() }; - let page = mind.query(&selection, None).unwrap(); + let page = mind.query(&selection).unwrap(); let edges = page.edges.expect("a hop carries edges"); let mut found: Vec<(CitationRole, String, String)> = edges.iter().map(|edge| (edge.role, edge.from.id.0.clone(), edge.to.id.0.clone())).collect(); @@ -1258,7 +1363,7 @@ mod tests { } } let history = Selection { cites: Some(cites(K::Question, &q1, Some("subject"))), ..of_kinds(&[K::Resolution]) }; - let page = mind.query(&history, None).unwrap(); + let page = mind.query(&history).unwrap(); let expected: Vec = (1..=10).map(|cycle| id("resolution", &format!("question.Q1.n{cycle}"))).collect(); assert_eq!(ids(&page), expected, "ordinal order, so n10 follows n9"); let statuses: Vec = @@ -1272,7 +1377,7 @@ mod tests { // resolutions by `subject`, the ten rulings by `answers` and the // follow-up by `source`. let unroled = Selection { cites: Some(cites(K::Question, &q1, None)), ..Selection::default() }; - let page = mind.query(&unroled, None).unwrap(); + let page = mind.query(&unroled).unwrap(); assert_eq!(page.matched, 21); assert!(page.edges.unwrap().iter().any(|edge| edge.role == CitationRole::Answers)); } @@ -1285,7 +1390,7 @@ mod tests { committed(admit(&mut mind, vec![hand_off(INSTANCE, OTHER_INSTANCE, REPO, &[])])); let assignments = |root: &str, repo_name: &str| { let selection = with(with(of_kinds(&[K::Stewardship]), any_of("root", &[root])), any_of("repo", &[repo_name])); - mind.query(&selection, None).unwrap() + mind.query(&selection).unwrap() }; let mine = assignments(INSTANCE, REPO); assert_eq!(ids(&mine), vec![format!("{INSTANCE}:stewardship:gamecult_-epiphany.n1")]); @@ -1342,7 +1447,7 @@ mod tests { cited: Some(Incoming { role: role.into(), exists: false }), ..selection }; - let open = |selection: Selection| ids(&mind.query(&selection, None).unwrap()); + let open = |selection: Selection| ids(&mind.query(&selection).unwrap()); assert_eq!(open(standing(mine(&[K::Question]))), vec![q1.clone()]); assert_eq!(open(standing(mine(&[K::Finding]))), vec![f1]); @@ -1357,54 +1462,168 @@ mod tests { assert_eq!(open(standing(of_kinds(&[K::Question]))).len(), 2); } - /// `semantic` is refused typed, before the image or the receipts are - /// read, and never answered with an empty page that reads like "nothing - /// matched". Where the check sits is the rule: below the reader, a store - /// the reader refuses would answer the integrity fault instead, which is - /// not what the caller asked about. - #[test] - fn semantic_query_refuses_typed_until_wired() { - let store = MemoryStore::new(); - let mut mind = opened(store.clone(), INSTANCE); - seed(&mut mind); - let q1 = id("question", "Q1"); - committed(admit(&mut mind, vec![question("Q1", &["A", "B"], "A")])); - let selection = with(with(of_kinds(&[K::Question]), any_of("root", &[CAMPAIGN])), any_of("in_force", &["true"])); - let semantic = SemanticQuery { text: "what did we rule about keys".into(), top_k: 5 }; - let unwired = MindRefusal::Unavailable { detail: "semantic query: the index is not wired (Cut 11)".into() }; - assert_eq!(mind.query(&selection, Some(&semantic)).err(), Some(unwired.clone())); - assert!(mind.query(&selection, None).is_ok()); - // A selection that is itself invalid still earns the semantic - // refusal first: nothing about it is read. - let malformed = with(Selection::default(), any_of("root", &["a/../b"])); - assert_eq!(mind.query(&malformed, Some(&semantic)).err(), Some(unwired.clone())); + fn hit(kind: K, key: &str, score: f32) -> (PipelineRef, f32) { + (r(kind, key), score) + } - // Two receipts naming one write is a store the reader refuses to build - // over at all. The semantic refusal still lands, because it is decided - // before the image and the receipts are read. - let doubled = MemoryStore::new(); - for row in store.rows() { - doubled.plant(row); + fn asking(top_k: u32) -> SemanticQuery { + SemanticQuery { text: "who owns the state".into(), top_k } + } + + fn standing(selection: Selection) -> Selection { + with(selection, any_of("in_force", &["true"])) + } + + /// A mind with every way a document stops being in force: Q1 answered + /// (resolved), Q2 answered and the answer withdrawn (its resolution is + /// resolved, the question in force again), R3 superseded by R4. + struct Shelf { + mind: Mind, + q1: String, + q2: String, + withdrawn_resolution: String, + r3: String, + r4: String, + } + + fn shelf() -> Shelf { + let mut mind = seeded(); + let (q1, q2) = (id("question", "Q1"), id("question", "Q2")); + committed(admit(&mut mind, vec![question("Q1", &["A", "B"], "A"), question("Q2", &["A", "B"], "A")])); + committed(admit(&mut mind, vec![answering("R1", &q1)])); + committed(admit(&mut mind, vec![answering("R2", &q2)])); + let withdrawn_resolution = id("resolution", "question.Q2.n1"); + committed(admit(&mut mind, vec![resolution(r(K::Resolution, &withdrawn_resolution), withdrawn())])); + let (r3, r4) = (id("ruling", "R3"), id("ruling", "R4")); + committed(admit(&mut mind, vec![D::Ruling(ruling("R3")), D::Ruling(ruling("R4"))])); + committed(admit(&mut mind, vec![resolution(r(K::Ruling, &r3), superseded(&[r(K::Ruling, &r4)]))])); + Shelf { mind, q1, q2, withdrawn_resolution, r3, r4 } + } + + /// The index never deletes and carries no status, so a hit is a candidate + /// and nothing more. Here it names a resolved question, a withdrawn + /// resolution, a superseded ruling, a document the mind does not hold and + /// a real id under the wrong kind, all scoring above the two documents + /// that stand; the standing ones are the whole answer. + #[test] + fn a_hit_is_judged_by_the_mind_and_never_by_the_index() { + let shelf = shelf(); + let hits = [ + hit(K::Question, &shelf.q1, 0.9), + hit(K::Resolution, &shelf.withdrawn_resolution, 0.8), + hit(K::Ruling, &shelf.r3, 0.7), + hit(K::Question, &id("question", "Q99"), 0.6), + hit(K::Question, &shelf.r4, 0.55), + hit(K::Ruling, &shelf.r4, 0.5), + hit(K::Question, &shelf.q2, 0.4), + ]; + let everything = shelf.mind.rank(&Selection::default(), &asking(10), &hits).unwrap(); + assert_eq!(ids(&everything), vec![shelf.q1.clone(), shelf.withdrawn_resolution.clone(), shelf.r3.clone(), shelf.r4.clone(), shelf.q2.clone()], + "without in_force the selection has not asked about standing, and the unheld and mis-kinded hits are gone"); + assert_eq!(everything.matched, 5); + + let page = shelf.mind.rank(&standing(Selection::default()), &asking(10), &hits).unwrap(); + assert_eq!(ids(&page), vec![shelf.r4.clone(), shelf.q2.clone()]); + assert_eq!((page.matched, page.next), (2, None)); + assert_eq!(page.as_of, receipt::head(&shelf.mind).unwrap()); + + let none = shelf.mind.rank(&standing(Selection::default()), &asking(10), &[hit(K::Question, &id("question", "Q99"), 1.0)]).unwrap(); + assert_eq!((none.matched, ids(&none)), (0, Vec::::new()), "an empty candidate set is an empty page, not a refusal"); + assert_eq!(shelf.mind.rank(&standing(Selection::default()), &asking(10), &[]).unwrap().matched, 0); + } + + /// The order is the score's, highest first, and ordinal only settles a + /// tie; the page is cut to the selection's limit and to top_k, and says + /// how many matched. + #[test] + fn ranked_results_are_ordered_by_score_then_ordinal_and_cut_to_the_limit() { + let shelf = shelf(); + let by_ordinal = [hit(K::Question, &shelf.q2, 0.4), hit(K::Ruling, &shelf.r4, 0.5)]; + assert_eq!(ids(&shelf.mind.rank(&standing(Selection::default()), &asking(10), &by_ordinal).unwrap()), vec![shelf.r4.clone(), shelf.q2.clone()], + "Q2 was admitted first, and the better score still comes first"); + let tied = [hit(K::Ruling, &shelf.r4, 0.5), hit(K::Question, &shelf.q2, 0.5)]; + assert_eq!(ids(&shelf.mind.rank(&standing(Selection::default()), &asking(10), &tied).unwrap()), vec![shelf.q2.clone(), shelf.r4.clone()], + "a tie falls to the earlier ordinal"); + + let limited = Selection { limit: Some(1), ..standing(Selection::default()) }; + let page = shelf.mind.rank(&limited, &asking(10), &by_ordinal).unwrap(); + assert_eq!((ids(&page), page.matched, page.next), (vec![shelf.r4.clone()], 2, None), "the limit keeps the best, not the earliest"); + let page = shelf.mind.rank(&standing(Selection::default()), &asking(1), &by_ordinal).unwrap(); + assert_eq!((ids(&page), page.matched), (vec![shelf.r4.clone()], 2), "top_k bounds the page too"); + } + + /// Membership is the selection's alone: a kind, a key allowlist and a + /// citation predicate each cut the candidates exactly as they cut a plain + /// query, and the citation predicate sees the whole mind, not only the + /// candidates (Q2 is named by a resolution that is not a hit). + #[test] + fn the_selections_own_predicates_decide_which_candidates_stay() { + let shelf = shelf(); + let hits = [hit(K::Question, &shelf.q2, 0.4), hit(K::Ruling, &shelf.r4, 0.5)]; + assert_eq!(ids(&shelf.mind.rank(&of_kinds(&[K::Ruling]), &asking(10), &hits).unwrap()), vec![shelf.r4.clone()]); + let keyed = Selection { keys: Some(vec![shelf.q2.clone(), shelf.q1.clone()]), ..Selection::default() }; + assert_eq!(ids(&shelf.mind.rank(&keyed, &asking(10), &hits).unwrap()), vec![shelf.q2.clone()], "keys intersect the candidates"); + let disjoint = Selection { keys: Some(vec![shelf.q1.clone()]), ..Selection::default() }; + assert_eq!(shelf.mind.rank(&disjoint, &asking(10), &hits).unwrap().matched, 0); + let unnamed = Selection { cited: Some(Incoming { role: "subject".into(), exists: false }), ..Selection::default() }; + let plain = ids(&shelf.mind.query(&Selection { keys: Some(vec![shelf.q2.clone(), shelf.r4.clone()]), ..unnamed.clone() }).unwrap()); + assert_eq!(plain, vec![shelf.r4.clone()], "no resolution names R4 as its subject, and one names Q2"); + assert_eq!(ids(&shelf.mind.rank(&unnamed, &asking(10), &hits).unwrap()), plain, "the same answer as the plain query over the same rows"); + assert_eq!(ids(&shelf.mind.rank(&Selection { projection: "document".into(), ..of_kinds(&[K::Ruling]) }, &asking(10), &hits).unwrap()), vec![shelf.r4.clone()]); + } + + /// A citation hop is answered over the whole mind, and its edges follow the + /// ranked page: after the cut to the limit only the edges anchored on a kept + /// row remain, in the page's order and not the evaluator's. An empty cursor + /// is no cursor. + #[test] + fn a_hop_keeps_only_the_edges_of_the_ranked_page_in_its_order() { + let shelf = shelf(); + let named = Selection { cited: Some(Incoming { role: "subject".into(), exists: true }), ..of_kinds(&[K::Question]) }; + let hits = [hit(K::Question, &shelf.q2, 0.9), hit(K::Question, &shelf.q1, 0.4)]; + let both = shelf.mind.rank(&named, &asking(10), &hits).unwrap(); + assert_eq!(ids(&both), vec![shelf.q2.clone(), shelf.q1.clone()], "Q1 was admitted first and scored lower"); + let edges = both.edges.expect("a hop carries its edges"); + let targets: Vec<&str> = edges.iter().map(|edge| edge.to.id.0.as_str()).collect(); + assert_eq!(targets, [shelf.q2.as_str(), shelf.q1.as_str()], "the edges follow the page order"); + assert!(edges.iter().all(|edge| edge.from.kind == K::Resolution && edge.role == CitationRole::Subject)); + + let limited = shelf.mind.rank(&Selection { limit: Some(1), ..named.clone() }, &asking(10), &hits).unwrap(); + assert_eq!(ids(&limited), vec![shelf.q2.clone()]); + let kept = limited.edges.unwrap(); + assert_eq!(kept.len(), 1, "the dropped row's edge is dropped with it"); + assert_eq!(kept[0].to.id.0, shelf.q2); + assert_eq!(kept[0].from.id.0, id("resolution", "question.Q2.n1")); + + let empty_cursor = Selection { cursor: Some(String::new()), ..standing(Selection::default()) }; + let with = shelf.mind.rank(&empty_cursor, &asking(10), &[hit(K::Ruling, &shelf.r4, 0.5)]).unwrap(); + assert_eq!(ids(&with), vec![shelf.r4.clone()]); + } + + + /// A ranked answer cannot be resumed, so a cursor is refused by name, and + /// so is a query that cannot be asked, before any hit is looked at. + #[test] + fn a_semantic_query_that_cannot_be_ranked_is_refused_by_name() { + let shelf = shelf(); + let hits = [hit(K::Ruling, &shelf.r4, 0.5)]; + let with_cursor = Selection { cursor: Some("anything".into()), ..Selection::default() }; + fn on(field: &str, refusal: &MindRefusal) -> bool { + matches!(refusal, MindRefusal::SelectionInvalid { field: f, .. } if f == field) } - let forged = crate::receipt::candidate( - &slug(INSTANCE), - provenance(Faculty::Soul), - &[], - &[prepare(&question("Q1", &["A", "B"], "A")), prepare(&question("Q8", &["A", "B"], "A"))], - 3, // head + 1: dense, so this plants only the duplicate identity, not an F1 density fault too. - now(), - ) - .unwrap(); - doubled.plant(schema_cache().unwrap().prepare_entry_named(&forged.receipt_id, &forged).unwrap().0); - let unreadable = opened(doubled, INSTANCE); - assert_eq!( - unreadable.query(&Selection::default(), None).err(), - Some(MindRefusal::Unavailable { - detail: format!("document {}/{q1} is written by two receipts", K::Question.type_id()), - }), - "the reader refuses this store" - ); - assert_eq!(unreadable.query(&selection, Some(&semantic)).err(), Some(unwired), "and the semantic refusal precedes reading it"); + for refusal in [ + shelf.mind.rank(&with_cursor, &asking(10), &hits).unwrap_err(), + shelf.mind.check_semantic(&with_cursor, &asking(10)).unwrap_err(), + ] { + assert!(on("cursor", &refusal), "{refusal:?}"); + } + assert!(on("semantic.top_k", &shelf.mind.rank(&Selection::default(), &asking(0), &hits).unwrap_err())); + assert!(on("semantic.top_k", &shelf.mind.rank(&Selection::default(), &asking(201), &hits).unwrap_err())); + let blank = SemanticQuery { text: " ".into(), top_k: 3 }; + assert!(on("semantic.text", &shelf.mind.rank(&Selection::default(), &blank, &hits).unwrap_err())); + let malformed = with(Selection::default(), any_of("root", &["a/../b"])); + assert_eq!(shelf.mind.check_semantic(&malformed, &asking(3)), shelf.mind.query(&malformed).map(drop), "the same door as a plain query"); + assert!(shelf.mind.check_semantic(&standing(Selection::default()), &asking(3)).is_ok()); } /// A summary is bounded by a named constant whatever the document's lists @@ -1452,7 +1671,7 @@ mod tests { }; committed(mind.admit(batch, now())); - let page = mind.query(&Selection::default(), None).unwrap(); + let page = mind.query(&Selection::default()).unwrap(); let sizes: Vec = headers(&page).iter().map(|summary| rmp_serde::to_vec_named(summary).unwrap().len()).collect(); let widest = *sizes.iter().max().unwrap(); diff --git a/schemas/cultnet/huginn.mind_request.v1.schema.json b/schemas/cultnet/huginn.mind_request.v1.schema.json index 9f02bbd..6d56bcb 100644 --- a/schemas/cultnet/huginn.mind_request.v1.schema.json +++ b/schemas/cultnet/huginn.mind_request.v1.schema.json @@ -1678,7 +1678,7 @@ ] }, "SemanticQuery": { - "description": "A semantic query, accepted by the type and refused until Cut 11 owns the\nindex. Cut 11 also owns which fields of each kind are text, so no substring\nfilter lives here to become a second answer to that question.", + "description": "A semantic query: the text to find documents near, and how many. The index\nowns which fields of each kind are text (`index_text`), so no substring\nfilter lives here to become a second answer to that question.", "type": "object", "properties": { "text": { From b692b901d2129b149abcc0492cd493c099a7debb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 14:07:35 +0200 Subject: [PATCH 012/520] Mind::rank leaves the cursor to the door that already refused it The narrowed selection cleared a cursor that semantic_selection has already refused, and an empty cursor is no cursor to the evaluator; the mutation run showed the clear could be deleted with every test still green. A test pins that an empty cursor still ranks. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-mind/src/query.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/huginn-mind/src/query.rs b/crates/huginn-mind/src/query.rs index a3d377b..87efa27 100644 --- a/crates/huginn-mind/src/query.rs +++ b/crates/huginn-mind/src/query.rs @@ -532,7 +532,7 @@ impl Mind { let mut evaluation = if keys.is_empty() { Evaluation { rows: Vec::new(), matched: 0, edges: Vec::new(), next_cursor: None } } else { - let narrowed = Selection { keys: Some(keys), limit: Some(LIMIT_MAX), cursor: None, ..selection.clone() }; + let narrowed = Selection { keys: Some(keys), limit: Some(LIMIT_MAX), ..selection.clone() }; select(&Vocabulary, &rows, &narrowed, head, self.cursor_key())? }; let score = |row: &SelectionRow| scores.get(row.key.as_str()).copied().unwrap_or(f32::NEG_INFINITY); From 7df718d643f2ef43d6204a5b4b2a9fe03882664e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:25:46 +0200 Subject: [PATCH 013/520] Mind::rank returns the refs it dropped; delete Daemon::unheld Daemon::unheld called mind.view once per hit, and every call decodes the whole mind: 3.27 s on the loop thread at 2000 documents and 200 hits, to feed one log line, and a second judge of whether the mind holds a document. rank already reads the rows once and is the only judge now: it returns the hits it found no document for in Ranked { page, dropped }, and Daemon::finish reports a count and the first five. Also in rank: candidates are filtered to held documents before they are cut to the best 200 by score, so an id the mind lacks cannot take a place from one it holds and select never sees more than it can answer (its own cut is by ordinal). A semantic selection with descending is refused SelectionInvalid, as cursor is: the answer's order is the score's and was silently ignored. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/daemon.rs | 22 ++-- crates/huginn-daemon/src/semantic_tests.rs | 20 ---- crates/huginn-mind/src/lib.rs | 2 +- crates/huginn-mind/src/query.rs | 123 ++++++++++++++++----- 4 files changed, 105 insertions(+), 62 deletions(-) diff --git a/crates/huginn-daemon/src/daemon.rs b/crates/huginn-daemon/src/daemon.rs index ceb86ef..51a9fe6 100644 --- a/crates/huginn-daemon/src/daemon.rs +++ b/crates/huginn-daemon/src/daemon.rs @@ -161,13 +161,6 @@ impl> Daemon { self.index.searched() } - /// The ids among `hits` that the mind holds no document for: the index - /// keeps points the mind no longer has (a restored mind, a second store), - /// and a hit is never shown on the index's word alone. - pub(crate) fn unheld<'a>(&self, hits: &'a Hits) -> Vec<&'a str> { - hits.iter().filter(|(id, _)| !matches!(self.mind.view(id), Ok(Some(_)))).map(|(id, _)| id.id.0.as_str()).collect() - } - /// The answer to a search: the index's candidates ranked through the mind /// (`Mind::rank` drops what the mind does not hold and lets the selection /// decide the rest), or, when the index could not answer, a typed @@ -177,11 +170,18 @@ impl> Daemon { Ok(hits) => hits, Err(detail) => return HuginnMindResponse::Refused(MindRefusal::Unavailable { detail }), }; - for id in self.unheld(&hits) { - eprintln!("huginn: the index returned {id}, which this mind does not hold; it is left out"); - } match self.mind.rank(&search.selection, &search.semantic, &hits) { - Ok(page) => HuginnMindResponse::Query(page), + Ok(ranked) => { + if !ranked.dropped.is_empty() { + let named: Vec<&str> = ranked.dropped.iter().take(5).map(|hit| hit.id.0.as_str()).collect(); + eprintln!( + "huginn: the index returned {} ids this mind does not hold (first {:?}); they are left out", + ranked.dropped.len(), + named + ); + } + HuginnMindResponse::Query(ranked.page) + } Err(refusal) => HuginnMindResponse::Refused(refusal), } } diff --git a/crates/huginn-daemon/src/semantic_tests.rs b/crates/huginn-daemon/src/semantic_tests.rs index 311a51e..ad8bb19 100644 --- a/crates/huginn-daemon/src/semantic_tests.rs +++ b/crates/huginn-daemon/src/semantic_tests.rs @@ -412,23 +412,3 @@ fn an_index_that_cannot_answer_is_unavailable_and_plain_queries_still_work() { assert!(detail.contains("refused") && detail.contains("no Huginn metadata"), "{detail}"); assert_eq!(documents(&mut client, "m-plain-3"), held); } - -/// The ids a daemon reports as unheld are exactly the hits its mind has no -/// document for: an absent id, and an id under a kind it is not. -#[test] -fn a_hit_the_mind_holds_no_document_for_is_named_unheld() { - let root = tempfile::tempdir().unwrap(); - let mut daemon = crate::daemon::tests::open_unindexed(root.path(), &slug(INSTANCE)).unwrap(); - let identity = crate::daemon::tests::identity(INSTANCE); - let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = - daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![identity])), now()).answered() - else { - panic!("the identity was not admitted") - }; - let held = writes[0].clone(); - let absent = PipelineRef { kind: held.kind, id: Short(id("instance", "nobody")) }; - let mis_kinded = PipelineRef { kind: PipelineKind::Question, id: held.id.clone() }; - let hits = vec![(held.clone(), 0.9), (absent.clone(), 0.8), (mis_kinded.clone(), 0.7)]; - assert_eq!(daemon.unheld(&hits), vec![absent.id.0.as_str(), mis_kinded.id.0.as_str()]); - assert!(daemon.unheld(&vec![(held, 0.9)]).is_empty()); -} diff --git a/crates/huginn-mind/src/lib.rs b/crates/huginn-mind/src/lib.rs index 8029548..0fd85e7 100644 --- a/crates/huginn-mind/src/lib.rs +++ b/crates/huginn-mind/src/lib.rs @@ -44,7 +44,7 @@ pub use admission::{BATCH_MAX, PipelineAdmissionBatch, PipelineAdmissionOutcome} pub use mind::{HuginnMindEpoch, Mind}; pub use query::{ AdmissionFacts, PipelineDocumentSummary, PipelineDocumentView, PipelineEdge, PipelineFacts, PipelinePageItems, - PipelineSelectionPage, PipelineStatus, PipelineStatusSummary, SUMMARY_MAX_BYTES, SemanticQuery, + PipelineSelectionPage, PipelineStatus, PipelineStatusSummary, Ranked, SUMMARY_MAX_BYTES, SemanticQuery, }; pub use receipt::{DocumentVersion, Faculty, HuginnCommitReceipt, PipelineProvenance, RECEIPT_SCHEMA_VERSION}; pub use refusal::MindRefusal; diff --git a/crates/huginn-mind/src/query.rs b/crates/huginn-mind/src/query.rs index 87efa27..274d902 100644 --- a/crates/huginn-mind/src/query.rs +++ b/crates/huginn-mind/src/query.rs @@ -275,6 +275,15 @@ pub struct PipelineSelectionPage { pub edges: Option>, } +/// A ranked semantic answer: the page, and the hits the index named that the +/// mind holds no document for (a restored mind, a second store). Local to the +/// process; it is never on the wire. +#[derive(Clone, Debug, PartialEq)] +pub struct Ranked { + pub page: PipelineSelectionPage, + pub dropped: Vec, +} + /// The admission facts of every stored document, built once per read call from /// the receipts. `writes` only: a document a later batch cited as a strong read /// keeps the receipt that wrote it. A10 admits one write per identity, so two @@ -455,8 +464,9 @@ impl Mind { } /// What a semantic query must satisfy before anything is embedded: the - /// selection passes the same two doors `query` does, carries no cursor - /// (a ranked answer is not pageable), and asks for between one and + /// selection passes the same two doors `query` does, carries no cursor and + /// no `descending` (a ranked answer is neither pageable nor reversible), + /// and asks for between one and /// `LIMIT_MAX` hits over text that says something. Returns the selection /// as the door leaves it. fn semantic_selection(selection: &Selection, semantic: &SemanticQuery) -> Result { @@ -468,6 +478,9 @@ impl Mind { if selection.cursor.as_deref().is_some_and(|cursor| !cursor.is_empty()) { return Err(refused("cursor", None, "a semantic answer is ranked, not paged: it carries no cursor")); } + if selection.descending { + return Err(refused("descending", Some("true".into()), "a semantic answer is ordered by score: it has no order to reverse")); + } if semantic.text.0.trim().is_empty() { return Err(refused("semantic.text", None, "a semantic query needs text")); } @@ -488,7 +501,8 @@ impl Mind { Self::semantic_selection(selection, semantic).map(drop) } - /// The page a semantic query answers, from the candidates the index gave. + /// The page a semantic query answers, from the candidates the index gave, + /// with the hits the mind holds no document for. /// /// The index supplies ids and scores and nothing else. A hit is a /// candidate: it is dropped unless the mind holds a document of that kind @@ -499,30 +513,44 @@ impl Mind { /// mean in `query`. Nothing about a document's standing is read from the /// index; `in_force` is the caller's to ask for and the mind's to derive. /// + /// This is the one place the mind judges whether it holds a hit, in the + /// one read of its rows that ranking needs: `Ranked::dropped` returns the + /// hits it found no document for, for the caller to report. + /// /// The order is the one thing the substrate cannot know: score, highest /// first, ties by ordinal. The page holds at most the selection's limit /// and at most `top_k` documents, `matched` counts every candidate the - /// selection passed, and `next` is never set. Only the best - /// `LIMIT_MAX` candidates are considered. + /// selection passed, and `next` is never set. The substrate answers at + /// most `LIMIT_MAX` rows and cuts by ordinal, so the candidates are cut + /// to the best `LIMIT_MAX` by score first, among the documents the mind + /// holds: an id the mind does not hold never takes the place of one it + /// does. pub fn rank( &self, selection: &Selection, semantic: &SemanticQuery, hits: &[(PipelineRef, f32)], - ) -> Result { + ) -> Result { let selection = Self::semantic_selection(selection, semantic)?; let head = receipt::head(self)?; let rows = Reader::at(self, head)?.rows()?; - let mut best: Vec<&(PipelineRef, f32)> = hits.iter().collect(); - best.sort_by(|a, b| b.1.total_cmp(&a.1)); - best.truncate(LIMIT_MAX as usize); - let mut scores: BTreeMap<&str, f32> = BTreeMap::new(); - for row in &rows { - if let Some((_, score)) = best.iter().find(|(hit, _)| hit.kind == row.kind && hit.id.0 == row.key) { - scores.insert(row.key.as_str(), *score); + let held: BTreeMap<(PipelineKind, &str), i64> = + rows.iter().map(|row| ((row.kind, row.key.as_str()), row.ordinal())).collect(); + let mut candidates: Vec<(&str, f32, i64)> = Vec::new(); + let mut dropped = Vec::new(); + for (hit, score) in hits { + match held.get(&(hit.kind, hit.id.0.as_str())) { + Some(&ordinal) => candidates.push((hit.id.0.as_str(), *score, ordinal)), + None => dropped.push(hit.clone()), } } + candidates.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.2.cmp(&b.2))); + candidates.truncate(LIMIT_MAX as usize); + let mut scores: BTreeMap<&str, f32> = BTreeMap::new(); + for (key, score, _) in &candidates { + scores.entry(key).or_insert(*score); + } let mut keys: Vec = scores.keys().map(|key| key.to_string()).collect(); if let Some(allowed) = &selection.keys { keys.retain(|key| allowed.contains(key)); @@ -553,7 +581,7 @@ impl Mind { .collect(); kept.sort_by_key(|(at, _)| *at); evaluation.edges = kept.into_iter().map(|(_, edge)| edge).collect(); - Self::page(&selection, evaluation, head) + Ok(Ranked { page: Self::page(&selection, evaluation, head)?, dropped }) } /// The projection of an evaluation: a header or a whole view per row, and @@ -1517,19 +1545,47 @@ mod tests { hit(K::Ruling, &shelf.r4, 0.5), hit(K::Question, &shelf.q2, 0.4), ]; - let everything = shelf.mind.rank(&Selection::default(), &asking(10), &hits).unwrap(); + let ranked = shelf.mind.rank(&Selection::default(), &asking(10), &hits).unwrap(); + assert_eq!( + ranked.dropped, + vec![r(K::Question, &id("question", "Q99")), r(K::Question, &shelf.r4)], + "the ids the mind holds nothing for come back, in the order the index gave them, and no held id does" + ); + let everything = ranked.page; assert_eq!(ids(&everything), vec![shelf.q1.clone(), shelf.withdrawn_resolution.clone(), shelf.r3.clone(), shelf.r4.clone(), shelf.q2.clone()], "without in_force the selection has not asked about standing, and the unheld and mis-kinded hits are gone"); assert_eq!(everything.matched, 5); - let page = shelf.mind.rank(&standing(Selection::default()), &asking(10), &hits).unwrap(); + let page = shelf.mind.rank(&standing(Selection::default()), &asking(10), &hits).unwrap().page; assert_eq!(ids(&page), vec![shelf.r4.clone(), shelf.q2.clone()]); assert_eq!((page.matched, page.next), (2, None)); assert_eq!(page.as_of, receipt::head(&shelf.mind).unwrap()); - let none = shelf.mind.rank(&standing(Selection::default()), &asking(10), &[hit(K::Question, &id("question", "Q99"), 1.0)]).unwrap(); + let none = shelf.mind.rank(&standing(Selection::default()), &asking(10), &[hit(K::Question, &id("question", "Q99"), 1.0)]).unwrap().page; assert_eq!((none.matched, ids(&none)), (0, Vec::::new()), "an empty candidate set is an empty page, not a refusal"); - assert_eq!(shelf.mind.rank(&standing(Selection::default()), &asking(10), &[]).unwrap().matched, 0); + assert_eq!(shelf.mind.rank(&standing(Selection::default()), &asking(10), &[]).unwrap().page.matched, 0); + } + + /// The substrate answers 200 rows and cuts by ordinal, so 250 candidates + /// must be cut by score first: the 200 best are answered, and the 50 lowest + /// scores are the ones that fall away, whichever of them was admitted first. + /// An id the mind does not hold does not take a place among the 200. + #[test] + fn more_candidates_than_the_substrate_answers_are_cut_by_score_not_by_ordinal() { + let mut mind = seeded(); + for start in (0..250).step_by(crate::admission::BATCH_MAX) { + committed(admit(&mut mind, (start..(start + crate::admission::BATCH_MAX as u32).min(250)).map(question_n).collect())); + } + let key = |n: u32| id("question", &format!("Q{n}")); + // Question n scores n, so the later a question was admitted the better it ranks. + let mut hits: Vec<(PipelineRef, f32)> = (0..250).map(|n| hit(K::Question, &key(n), n as f32)).collect(); + // Twenty ids the mind holds nothing for, above every real score. + hits.extend((0..20).map(|n| hit(K::Question, &id("question", &format!("Gone{n}")), 1000.0 + n as f32))); + let ranked = mind.rank(&Selection::default(), &asking(200), &hits).unwrap(); + let expected: Vec = (50..250).rev().map(key).collect(); + assert_eq!(ids(&ranked.page), expected, "the 200 best, best first"); + assert_eq!(ranked.page.matched, 200); + assert_eq!(ranked.dropped.len(), 20); } /// The order is the score's, highest first, and ordinal only settles a @@ -1539,16 +1595,16 @@ mod tests { fn ranked_results_are_ordered_by_score_then_ordinal_and_cut_to_the_limit() { let shelf = shelf(); let by_ordinal = [hit(K::Question, &shelf.q2, 0.4), hit(K::Ruling, &shelf.r4, 0.5)]; - assert_eq!(ids(&shelf.mind.rank(&standing(Selection::default()), &asking(10), &by_ordinal).unwrap()), vec![shelf.r4.clone(), shelf.q2.clone()], + assert_eq!(ids(&shelf.mind.rank(&standing(Selection::default()), &asking(10), &by_ordinal).unwrap().page), vec![shelf.r4.clone(), shelf.q2.clone()], "Q2 was admitted first, and the better score still comes first"); let tied = [hit(K::Ruling, &shelf.r4, 0.5), hit(K::Question, &shelf.q2, 0.5)]; - assert_eq!(ids(&shelf.mind.rank(&standing(Selection::default()), &asking(10), &tied).unwrap()), vec![shelf.q2.clone(), shelf.r4.clone()], + assert_eq!(ids(&shelf.mind.rank(&standing(Selection::default()), &asking(10), &tied).unwrap().page), vec![shelf.q2.clone(), shelf.r4.clone()], "a tie falls to the earlier ordinal"); let limited = Selection { limit: Some(1), ..standing(Selection::default()) }; - let page = shelf.mind.rank(&limited, &asking(10), &by_ordinal).unwrap(); + let page = shelf.mind.rank(&limited, &asking(10), &by_ordinal).unwrap().page; assert_eq!((ids(&page), page.matched, page.next), (vec![shelf.r4.clone()], 2, None), "the limit keeps the best, not the earliest"); - let page = shelf.mind.rank(&standing(Selection::default()), &asking(1), &by_ordinal).unwrap(); + let page = shelf.mind.rank(&standing(Selection::default()), &asking(1), &by_ordinal).unwrap().page; assert_eq!((ids(&page), page.matched), (vec![shelf.r4.clone()], 2), "top_k bounds the page too"); } @@ -1560,16 +1616,16 @@ mod tests { fn the_selections_own_predicates_decide_which_candidates_stay() { let shelf = shelf(); let hits = [hit(K::Question, &shelf.q2, 0.4), hit(K::Ruling, &shelf.r4, 0.5)]; - assert_eq!(ids(&shelf.mind.rank(&of_kinds(&[K::Ruling]), &asking(10), &hits).unwrap()), vec![shelf.r4.clone()]); + assert_eq!(ids(&shelf.mind.rank(&of_kinds(&[K::Ruling]), &asking(10), &hits).unwrap().page), vec![shelf.r4.clone()]); let keyed = Selection { keys: Some(vec![shelf.q2.clone(), shelf.q1.clone()]), ..Selection::default() }; - assert_eq!(ids(&shelf.mind.rank(&keyed, &asking(10), &hits).unwrap()), vec![shelf.q2.clone()], "keys intersect the candidates"); + assert_eq!(ids(&shelf.mind.rank(&keyed, &asking(10), &hits).unwrap().page), vec![shelf.q2.clone()], "keys intersect the candidates"); let disjoint = Selection { keys: Some(vec![shelf.q1.clone()]), ..Selection::default() }; - assert_eq!(shelf.mind.rank(&disjoint, &asking(10), &hits).unwrap().matched, 0); + assert_eq!(shelf.mind.rank(&disjoint, &asking(10), &hits).unwrap().page.matched, 0); let unnamed = Selection { cited: Some(Incoming { role: "subject".into(), exists: false }), ..Selection::default() }; let plain = ids(&shelf.mind.query(&Selection { keys: Some(vec![shelf.q2.clone(), shelf.r4.clone()]), ..unnamed.clone() }).unwrap()); assert_eq!(plain, vec![shelf.r4.clone()], "no resolution names R4 as its subject, and one names Q2"); - assert_eq!(ids(&shelf.mind.rank(&unnamed, &asking(10), &hits).unwrap()), plain, "the same answer as the plain query over the same rows"); - assert_eq!(ids(&shelf.mind.rank(&Selection { projection: "document".into(), ..of_kinds(&[K::Ruling]) }, &asking(10), &hits).unwrap()), vec![shelf.r4.clone()]); + assert_eq!(ids(&shelf.mind.rank(&unnamed, &asking(10), &hits).unwrap().page), plain, "the same answer as the plain query over the same rows"); + assert_eq!(ids(&shelf.mind.rank(&Selection { projection: "document".into(), ..of_kinds(&[K::Ruling]) }, &asking(10), &hits).unwrap().page), vec![shelf.r4.clone()]); } /// A citation hop is answered over the whole mind, and its edges follow the @@ -1581,14 +1637,14 @@ mod tests { let shelf = shelf(); let named = Selection { cited: Some(Incoming { role: "subject".into(), exists: true }), ..of_kinds(&[K::Question]) }; let hits = [hit(K::Question, &shelf.q2, 0.9), hit(K::Question, &shelf.q1, 0.4)]; - let both = shelf.mind.rank(&named, &asking(10), &hits).unwrap(); + let both = shelf.mind.rank(&named, &asking(10), &hits).unwrap().page; assert_eq!(ids(&both), vec![shelf.q2.clone(), shelf.q1.clone()], "Q1 was admitted first and scored lower"); let edges = both.edges.expect("a hop carries its edges"); let targets: Vec<&str> = edges.iter().map(|edge| edge.to.id.0.as_str()).collect(); assert_eq!(targets, [shelf.q2.as_str(), shelf.q1.as_str()], "the edges follow the page order"); assert!(edges.iter().all(|edge| edge.from.kind == K::Resolution && edge.role == CitationRole::Subject)); - let limited = shelf.mind.rank(&Selection { limit: Some(1), ..named.clone() }, &asking(10), &hits).unwrap(); + let limited = shelf.mind.rank(&Selection { limit: Some(1), ..named.clone() }, &asking(10), &hits).unwrap().page; assert_eq!(ids(&limited), vec![shelf.q2.clone()]); let kept = limited.edges.unwrap(); assert_eq!(kept.len(), 1, "the dropped row's edge is dropped with it"); @@ -1596,7 +1652,7 @@ mod tests { assert_eq!(kept[0].from.id.0, id("resolution", "question.Q2.n1")); let empty_cursor = Selection { cursor: Some(String::new()), ..standing(Selection::default()) }; - let with = shelf.mind.rank(&empty_cursor, &asking(10), &[hit(K::Ruling, &shelf.r4, 0.5)]).unwrap(); + let with = shelf.mind.rank(&empty_cursor, &asking(10), &[hit(K::Ruling, &shelf.r4, 0.5)]).unwrap().page; assert_eq!(ids(&with), vec![shelf.r4.clone()]); } @@ -1617,6 +1673,13 @@ mod tests { ] { assert!(on("cursor", &refusal), "{refusal:?}"); } + let reversed = Selection { descending: true, ..Selection::default() }; + for refusal in [ + shelf.mind.rank(&reversed, &asking(10), &hits).unwrap_err(), + shelf.mind.check_semantic(&reversed, &asking(10)).unwrap_err(), + ] { + assert!(on("descending", &refusal), "a ranked answer has no order to reverse: {refusal:?}"); + } assert!(on("semantic.top_k", &shelf.mind.rank(&Selection::default(), &asking(0), &hits).unwrap_err())); assert!(on("semantic.top_k", &shelf.mind.rank(&Selection::default(), &asking(201), &hits).unwrap_err())); let blank = SemanticQuery { text: " ".into(), top_k: 3 }; From 4b5eaf9b8fc83c065a25f7e937140eb24b465f7f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:25:55 +0200 Subject: [PATCH 014/520] Semantic search refuses while the index rebuilds; bound, cancel and re-verify Operator ruling 2026-09-30: while the index is rebuilding (a re-pulled model, a lost collection, Reconciling or Behind) a semantic search refuses with a typed Unavailable naming the state and the pending count, at the sink and again in Projector::search, and never answers from a half-filled collection. Soul's findings on 11b: - a failed search clears the reconciliation, so whoami stops reporting Current and the worker verifies again; the idle recheck is an absolute time that searches no longer push out; - the queue holds at most SEARCHES_QUEUED_MAX and refuses the rest by name; the worker serves one search per turn and returns to its writes; the serve loop takes back a search it timed out (IndexSink::abandon), so a queued one is never embedded; - the model identity is read again after the query is embedded, and the collection's label is re-read before it: a search is one decision; - a failure report's pending count is refreshed under the lock that takes the inbox (absorb); - wait_status has one overall deadline, and the test gate and worker-stop waits are bounded, so a mutant fails instead of hanging. Tests pin each rule: ticket uniqueness, searched() returning exactly what the worker answered, the refuse_searches call site, serving between batches, the bound, abandonment, the rechecks and the label. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/daemon.rs | 15 + crates/huginn-daemon/src/index/fakes.rs | 13 +- crates/huginn-daemon/src/index/mod.rs | 297 +++++++++----- crates/huginn-daemon/src/index/tests.rs | 441 ++++++++++++++++++++- crates/huginn-daemon/src/semantic_tests.rs | 71 +++- crates/huginn-daemon/src/serve.rs | 4 +- 6 files changed, 718 insertions(+), 123 deletions(-) diff --git a/crates/huginn-daemon/src/daemon.rs b/crates/huginn-daemon/src/daemon.rs index 51a9fe6..c42d644 100644 --- a/crates/huginn-daemon/src/daemon.rs +++ b/crates/huginn-daemon/src/daemon.rs @@ -28,6 +28,10 @@ pub trait IndexSink { /// Every search that finished since the last call, by ticket: its /// candidates and scores, or why it failed. fn searched(&mut self) -> Vec<(SearchTicket, Result)>; + /// The asker has stopped waiting for `ticket`. A search not yet started is + /// dropped, so a queue of searches nobody wants cannot hold the index's + /// writes off. + fn abandon(&mut self, ticket: SearchTicket); } /// Names one search the index was asked, so the loop can match the answer to @@ -161,6 +165,11 @@ impl> Daemon { self.index.searched() } + /// The client has stopped waiting for `ticket`. + pub fn abandon(&mut self, ticket: SearchTicket) { + self.index.abandon(ticket); + } + /// The answer to a search: the index's candidates ranked through the mind /// (`Mind::rank` drops what the mind does not hold and lets the selection /// decide the rest), or, when the index could not answer, a typed @@ -438,6 +447,8 @@ pub(crate) mod tests { fn searched(&mut self) -> Vec<(SearchTicket, Result)> { Vec::new() } + + fn abandon(&mut self, _ticket: SearchTicket) {} } /// A daemon over the instance's mind under `state_root`, with no index. @@ -472,6 +483,8 @@ pub(crate) mod tests { fn searched(&mut self) -> Vec<(SearchTicket, Result)> { Vec::new() } + + fn abandon(&mut self, _ticket: SearchTicket) {} } struct FailingIndex; @@ -492,6 +505,8 @@ pub(crate) mod tests { fn searched(&mut self) -> Vec<(SearchTicket, Result)> { Vec::new() } + + fn abandon(&mut self, _ticket: SearchTicket) {} } fn status(daemon: &mut Daemon>) -> MindStatus { diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs index d7e4eee..3ed7560 100644 --- a/crates/huginn-daemon/src/index/fakes.rs +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -24,9 +24,12 @@ impl Gate { /// Blocks until `count` calls to `embed` have arrived at the gate. pub(crate) fn wait_arrived(&self, count: u32) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); let mut state = self.0.0.lock().unwrap(); while state.0 < count { - state = self.0.1.wait(state).unwrap(); + let left = deadline.saturating_duration_since(std::time::Instant::now()); + assert!(!left.is_zero(), "{count} calls never arrived at the gate; {} did", state.0); + state = self.0.1.wait_timeout(state, left).unwrap().0; } } @@ -121,6 +124,10 @@ pub(crate) struct IndexState { pub(crate) hits: Vec, /// The collection, vector length and limit of every search, in order. pub(crate) searches: Vec<(String, usize, u32)>, + /// How many times every point id was listed: only a reconciliation does. + pub(crate) listed: u32, + /// Whether a search fails at the vector store, the rest of it working. + pub(crate) fail_search: bool, } #[derive(Clone, Default)] @@ -162,10 +169,11 @@ impl VectorIndex for FakeIndex { } fn ids(&mut self, collection: &str) -> Result> { - let state = self.state.lock().unwrap(); + let mut state = self.state.lock().unwrap(); if state.down { bail!("the vector store is down"); } + state.listed += 1; Ok(state.collections.get(collection).map(|stored| stored.points.keys().cloned().collect()).unwrap_or_default()) } @@ -188,6 +196,7 @@ impl VectorIndex for FakeIndex { bail!("the vector store is down"); } state.searches.push((collection.into(), vector.len(), limit)); + ensure!(!state.fail_search, "the vector store failed the search"); ensure!(state.collections.contains_key(collection), "no collection {collection}"); Ok(state.hits.clone()) } diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs index 9a943ba..1d7aaa6 100644 --- a/crates/huginn-daemon/src/index/mod.rs +++ b/crates/huginn-daemon/src/index/mod.rs @@ -26,10 +26,10 @@ //! re-reads the model's identity (name, digest, dimensions; about 0.3 s against //! Ollama) and, if it is not the one the collection was reconciled under, //! forgets the reconciliation instead of writing: a model re-pulled under the -//! same dimensions never mixes its vectors with the old model's. An idle -//! worker forgets the reconciliation every `Backoff::recheck` (60 s by -//! default), which re-reads the model, describes the collection and lists its -//! points: a collection lost while nothing was being written is found within +//! same dimensions never mixes its vectors with the old model's. The worker +//! forgets the reconciliation every `Backoff::recheck` (60 s by default) on a +//! schedule that neither admissions nor searches move, which re-reads the model, +//! describes the collection and lists its points: a collection lost while nothing was being written is found within //! that interval, and until then `Current` is what the worker last verified. //! There is one judge either way: `reconcile`. Refilling needs //! the text, so the projector keeps every entry it was given (`known`) beside @@ -60,9 +60,19 @@ //! them back through `Mind::rank`, which drops ids the mind does not hold and //! lets the selection decide the rest, in-force derivation included. //! - A search runs on the worker between batches, so it waits for the batch in -//! flight. The serve loop never waits: it holds the reply and answers when -//! the worker has (`ServeOptions::search_timeout` bounds the wait). A search -//! is refused at once while the worker is `Failing` or `Refused`. +//! flight, and at most one runs between two steps of the writes. The serve +//! loop never waits: it holds the reply and answers when the worker has +//! (`ServeOptions::search_timeout` bounds the wait), and takes back a search +//! it has stopped waiting for. At most `SEARCHES_QUEUED_MAX` wait; one more is +//! refused. A search whose embed is already running cannot be recalled, so it +//! can hold the writes off for as long as Ollama's own timeout. +//! - Only a current index answers a search. While the worker is `Reconciling` +//! or `Behind` (a re-pulled model, a lost collection, entries not yet +//! written), `Failing` or `Refused`, a search is refused at once with an +//! `Unavailable` that names the state and how many documents are pending; it +//! is never answered from a half-filled collection. A search that fails after +//! it was accepted forgets the reconciliation, so `whoami` stops reporting +//! `Current` and the worker verifies the model and the collection again. pub mod ollama; pub mod qdrant; @@ -339,15 +349,7 @@ impl Projector { return Err(StepError::Unavailable(anyhow::anyhow!("the mind has no identity yet"))); }; let identity = self.embedder.model_identity().context("reading the embedding model's identity")?; - let wanted = CollectionMeta { - managed_by: MANAGED_BY.into(), - instance: self.instance.clone(), - mind: mind.clone(), - model: identity.name.clone(), - model_digest: identity.digest.clone(), - dimensions: identity.dimensions, - index_text_version: INDEX_TEXT_VERSION, - }; + let wanted = self.collection_meta(&mind, &identity); match self.index.describe(&self.collection).context("describing the collection")? { Described::Absent => self.recreate(&wanted)?, Described::Unlabelled => { @@ -373,6 +375,20 @@ impl Projector { Ok(()) } + /// What the collection must say about itself to be this mind's, under + /// this model. + fn collection_meta(&self, mind: &str, identity: &ModelIdentity) -> CollectionMeta { + CollectionMeta { + managed_by: MANAGED_BY.into(), + instance: self.instance.clone(), + mind: mind.to_owned(), + model: identity.name.clone(), + model_digest: identity.digest.clone(), + dimensions: identity.dimensions, + index_text_version: INDEX_TEXT_VERSION, + } + } + fn recreate(&mut self, meta: &CollectionMeta) -> Result<()> { self.index.recreate(&self.collection, meta).context("creating the collection") } @@ -424,22 +440,56 @@ impl Projector { /// The candidates a query text finds: the text is embedded with the model's /// query instruction and the collection is searched, with no filter and an /// oversample of `min(top_k * 4, 200)`, because the mind will drop most of - /// what the index cannot know is not in force. A model that has changed - /// since the reconciliation refuses the search rather than compare vectors - /// of two models. A hit naming a kind this organ has no name for is logged - /// and left out. + /// what the index cannot know is not in force. A hit naming a kind this + /// organ has no name for is logged and left out. + /// + /// A search is answered only from an index that is current: while it is + /// being reconciled or written to, it refuses, naming why and how many + /// documents are pending, rather than answer from a half-filled + /// collection. It is also answered only from a collection that is still + /// this mind's, built by the model that is still the embedder: the model's + /// identity is read before and after the query is embedded and the + /// collection's label before it, and the query is refused if either + /// disagrees with what the collection was reconciled under. Any failure + /// after that forgets the reconciliation, so `progress_status` stops + /// reporting `Current` and the worker verifies again before it answers. pub fn search(&mut self, text: &str, top_k: u32) -> Result { - let Some(dimensions) = self.reconciled.as_ref().map(|model| model.dimensions as usize) else { - bail!("the index has not been reconciled with the model and the collection"); + if let Some(why) = unsearchable(&self.progress_status()) { + bail!("{why}"); + } + let found = self.nearest(text, top_k); + if found.is_err() { + self.reconciled = None; + } + found + } + + fn nearest(&mut self, text: &str, top_k: u32) -> Result { + let (Some(mind), Some(reconciled)) = (self.mind.clone(), self.reconciled.clone()) else { + bail!("the mind has no identity yet, so it has no index"); }; + let dimensions = reconciled.dimensions as usize; ensure!( self.model_unchanged().map_err(|error| anyhow::anyhow!("{error}"))?, "the embedding model changed since the collection was built; the index is being rebuilt for it" ); + let described = self.index.describe(&self.collection).context("describing the collection")?; + ensure!( + described == Described::Labelled(self.collection_meta(&mind, &reconciled)), + "collection {} is no longer the one this mind's index was built as: {described:?}", + self.collection + ); let query = format!("Instruct: {QUERY_INSTRUCTION}\nQuery: {text}"); let mut vectors = self.embedder.embed(&[query]).context("embedding the query")?; ensure!(vectors.len() == 1, "the embedder returned {} vectors for one query", vectors.len()); let vector = vectors.remove(0); + // The model is read again after the query was embedded: only a vector + // made between two reads of the same model is compared with the + // collection. + ensure!( + self.model_unchanged().map_err(|error| anyhow::anyhow!("{error}"))?, + "the embedding model changed while the query was embedded; the index is being rebuilt for it" + ); ensure!(vector.len() == dimensions, "the embedder returned a query vector of {} where the collection holds {dimensions}", vector.len()); let found = self.index.search(&self.collection, &vector, top_k.saturating_mul(4).min(OVERSAMPLE_MAX))?; let mut hits = Vec::with_capacity(found.len()); @@ -495,13 +545,20 @@ impl Backoff { } } +/// The most searches the worker holds waiting. A search asked when this many +/// wait is refused, by name, at once: the queue is bounded by the asker being +/// told, not by the worker's patience. +pub const SEARCHES_QUEUED_MAX: usize = 16; + struct Shared { inbox: Vec, /// The mind's identity, once a commit has revealed it to a mind that had none. mind: Option, status: IndexStatus, closed: bool, - /// Searches started and not yet taken by the worker. + /// Searches started and not yet taken by the worker, at most + /// `SEARCHES_QUEUED_MAX`; the serve loop takes back one it has stopped + /// waiting for. searches: VecDeque<(SearchTicket, String, u32)>, /// Searches the worker finished and the serve loop has not collected. found: Vec<(SearchTicket, Result)>, @@ -517,31 +574,30 @@ fn lock(shared: &Lock) -> MutexGuard<'_, Shared> { /// How long the worker waits, and what ends the wait early. enum Wait { /// Idle: until something arrives (an entry, the mind's identity, a search) - /// or `recheck` passes. - Work { recheck: Duration }, + /// or `until`. The deadline is the caller's, fixed when it was set: work + /// that wakes the worker does not move it. + Work { until: Instant }, /// Backing off from a failure: the whole delay, whatever arrives. A failing /// index is retried on its schedule, not once per admission. Backoff(Duration), } -/// Whether the wait ended because the recheck interval passed. -fn wait_for_work(shared: &Lock, wait: Wait) -> bool { - let (timeout, wakes_on_work) = match wait { - Wait::Work { recheck } => (recheck, true), - Wait::Backoff(delay) => (delay, false), +fn wait_for_work(shared: &Lock, wait: Wait) { + let (deadline, wakes_on_work) = match wait { + Wait::Work { until } => (until, true), + Wait::Backoff(delay) => (Instant::now() + delay, false), }; - let deadline = Instant::now() + timeout; let mut guard = lock(shared); loop { if guard.closed { - return false; + return; } if wakes_on_work && (!guard.inbox.is_empty() || guard.mind.is_some() || !guard.searches.is_empty()) { - return false; + return; } let left = deadline.saturating_duration_since(Instant::now()); if left.is_zero() { - return wakes_on_work; + return; } guard = shared.1.wait_timeout(guard, left).unwrap_or_else(|poisoned| poisoned.into_inner()).0; } @@ -570,16 +626,33 @@ fn including(status: &IndexStatus, queued: u32) -> IndexStatus { } } -/// Runs every search the serve loop has queued, one at a time, and leaves each -/// answer for the loop to collect. Between batches, never inside one. -fn serve_searches(projector: &mut Projector, shared: &Lock) { - loop { - let Some((ticket, text, top_k)) = lock(shared).searches.pop_front() else { return }; - let found = projector.search(&text, top_k).map_err(|error| format!("{error:#}")); - lock(shared).found.push((ticket, found)); +/// Why a status cannot answer a search, or `None` when it can. Only a current +/// index answers: one that is being reconciled or written to holds a +/// half-filled collection, and one that is failing or refused cannot be asked. +/// The reason names the state and how many documents are pending. +fn unsearchable(status: &IndexStatus) -> Option { + use IndexStatus as S; + match status { + S::Current => None, + S::Reconciling { pending } => Some(format!( + "the semantic index is being rebuilt: reconciling with the model and the collection (pending: {pending})" + )), + S::Behind { pending } => Some(format!("the semantic index is being rebuilt: behind the mind (pending: {pending})")), + S::Failing { pending, error, .. } => Some(format!("the semantic index is failing: {error} (pending: {pending})")), + S::Refused { pending, reason, .. } => Some(format!("the semantic index is refused: {reason} (pending: {pending})")), } } +/// Runs the search at the front of the queue, if any, and leaves its answer +/// for the loop to collect. Between batches, never inside one, and only one: +/// the worker returns to its writes after each, so a queue of searches cannot +/// hold the index's writes off for the sum of their embeddings. +fn serve_search(projector: &mut Projector, shared: &Lock) { + let Some((ticket, text, top_k)) = lock(shared).searches.pop_front() else { return }; + let found = projector.search(&text, top_k).map_err(|error| format!("{error:#}")); + lock(shared).found.push((ticket, found)); +} + /// Answers every queued search with why the index cannot: they were accepted /// while it was healthy and it has failed since. fn refuse_searches(shared: &Lock, why: &str) { @@ -589,63 +662,77 @@ fn refuse_searches(shared: &Lock, why: &str) { } } +/// The worker's take of what arrived since it last looked: the inbox is taken, +/// absorbed and its effect published under one lock. Between taking entries +/// out of the inbox and publishing them as wanted, `status` would otherwise +/// read a stale count with nothing queued, and a reader would see the mind +/// caught up (or a failure short of what it is failing to write) while +/// entries were in the worker's hands. `failing` is the failure being +/// reported and how many attempts it has had: the report stands, with the new +/// pending count, until an attempt succeeds. Returns false once the sink has +/// closed. +fn absorb( + projector: &mut Projector, + shared: &Lock, + failing: Option<(&StepError, u32)>, +) -> bool { + let mut guard = lock(shared); + if guard.closed { + return false; + } + let mind = guard.mind.take(); + let inbox = std::mem::take(&mut guard.inbox); + if let Some(mind) = mind { + projector.identify(mind); + } + projector.want(inbox); + guard.status = match failing { + None => projector.progress_status(), + Some((error, attempts)) => projector.failure_status(error, attempts), + }; + true +} + fn work(mut projector: Projector, shared: Lock, backoff: Backoff) { - let mut failures = 0_u32; + let mut failing: Option<(StepError, u32)> = None; let mut delay = backoff.initial; - let mut recheck = false; + // When the worker next verifies the model and the collection whether or + // not anything has woken it: an absolute time, moved only by a + // verification, so a steady stream of searches cannot keep pushing it out. + let mut recheck_at = Instant::now() + backoff.recheck; loop { - // The inbox is taken, absorbed and its effect published under one - // lock: between taking entries out of the inbox and publishing them - // as wanted, `status` would otherwise read a stale `Current` with - // nothing queued, and a reader would see the mind caught up while - // entries were in the worker's hands. - { - let mut guard = lock(&shared); - if guard.closed { - return; - } - let mind = guard.mind.take(); - let inbox = std::mem::take(&mut guard.inbox); - if let Some(mind) = mind { - projector.identify(mind); - } - projector.want(inbox); - // While a failure is being reported, the report stands until an - // attempt succeeds: publishing progress between retries would - // flicker. - if failures == 0 { - guard.status = projector.progress_status(); - } + if !absorb(&mut projector, &shared, failing.as_ref().map(|(error, attempts)| (error, *attempts))) { + return; } shared.1.notify_all(); - let stepped = if std::mem::take(&mut recheck) { projector.recheck() } else { projector.advance() }; + let due = Instant::now() >= recheck_at; + let verifies = due || projector.stale(); + let stepped = if due { projector.recheck() } else { projector.advance() }; match stepped { - Ok(Advance::Progressed) => { - failures = 0; + Ok(advance) => { + failing = None; delay = backoff.initial; + if verifies { + recheck_at = Instant::now() + backoff.recheck; + } publish(&shared, projector.progress_status()); - serve_searches(&mut projector, &shared); - } - Ok(Advance::Idle) => { - failures = 0; - delay = backoff.initial; - publish(&shared, projector.progress_status()); - serve_searches(&mut projector, &shared); - // A search that found the model changed has left the - // reconciliation to redo: do it now, not at the next wake. - if !projector.stale() { - recheck = wait_for_work(&shared, Wait::Work { recheck: backoff.recheck }); + serve_search(&mut projector, &shared); + // A search that failed has left the reconciliation to redo: + // do it now, not at the next wake. + if advance == Advance::Idle && !projector.stale() { + wait_for_work(&shared, Wait::Work { until: recheck_at }); } } Err(error) => { - failures = failures.saturating_add(1); + let attempts = failing.as_ref().map_or(0, |(_, attempts)| *attempts).saturating_add(1); eprintln!( - "huginn: the index could not advance (attempt {failures}, {} pending, retrying in {:?}): {error}", + "huginn: the index could not advance (attempt {attempts}, {} pending, retrying in {:?}): {error}", projector.pending(), delay ); - publish(&shared, projector.failure_status(&error, failures)); + publish(&shared, projector.failure_status(&error, attempts)); refuse_searches(&shared, &format!("the index could not advance: {error}")); + failing = Some((error, attempts)); wait_for_work(&shared, Wait::Backoff(delay)); delay = backoff.after(delay); } @@ -702,25 +789,30 @@ impl WorkerSink { /// Blocks until the health satisfies `predicate`, and returns it. For /// tests and probes that must observe the worker without a clock; it gives - /// up after two minutes rather than hang a run. + /// up after `WAIT_STATUS_MAX` in all, however often the health changes, + /// rather than hang a run. pub fn wait_status(&self, predicate: impl Fn(&IndexStatus) -> bool) -> IndexStatus { + self.wait_status_for(WAIT_STATUS_MAX, predicate) + } + + fn wait_status_for(&self, limit: Duration, predicate: impl Fn(&IndexStatus) -> bool) -> IndexStatus { + let deadline = Instant::now() + limit; let mut guard = lock(&self.shared); loop { let seen = including(&guard.status, u32::try_from(guard.inbox.len()).unwrap_or(u32::MAX)); if predicate(&seen) { return seen; } - let (next, result) = self - .shared - .1 - .wait_timeout(guard, Duration::from_secs(120)) - .unwrap_or_else(|poisoned| poisoned.into_inner()); - guard = next; - assert!(!result.timed_out(), "the index never reached the awaited state; last seen {:?}", guard.status); + let left = deadline.saturating_duration_since(Instant::now()); + assert!(!left.is_zero(), "the index never reached the awaited state; last seen {:?}", guard.status); + guard = self.shared.1.wait_timeout(guard, left).unwrap_or_else(|poisoned| poisoned.into_inner()).0; } } } +/// The longest `WorkerSink::wait_status` waits, in all. +const WAIT_STATUS_MAX: Duration = Duration::from_secs(30); + impl Drop for WorkerSink { fn drop(&mut self) { lock(&self.shared).closed = true; @@ -751,14 +843,20 @@ impl IndexSink for WorkerSink { including(&guard.status, u32::try_from(guard.inbox.len()).unwrap_or(u32::MAX)) } - /// Queues the search for the worker and returns its ticket. A worker that - /// is failing or refused is not asked: its own report is the refusal. + /// Queues the search for the worker and returns its ticket. An index that + /// is not current is not asked, and neither is a full queue: the refusal + /// is the answer, naming why and how many documents are pending. fn search(&mut self, text: &str, top_k: u32) -> Result { let mut guard = lock(&self.shared); - match &guard.status { - IndexStatus::Failing { error, .. } => return Err(format!("the semantic index is failing: {error}")), - IndexStatus::Refused { reason, .. } => return Err(format!("the semantic index is refused: {reason}")), - _ => {} + let status = including(&guard.status, u32::try_from(guard.inbox.len()).unwrap_or(u32::MAX)); + if let Some(why) = unsearchable(&status) { + return Err(why); + } + if guard.searches.len() >= SEARCHES_QUEUED_MAX { + return Err(format!( + "the semantic index has {} searches waiting and takes no more until it has answered some", + guard.searches.len() + )); } let ticket = SearchTicket(guard.next_ticket); guard.next_ticket += 1; @@ -771,6 +869,13 @@ impl IndexSink for WorkerSink { fn searched(&mut self) -> Vec<(SearchTicket, Result)> { std::mem::take(&mut lock(&self.shared).found) } + + /// A search still queued is taken out of the queue and never embedded. One + /// the worker already holds cannot be recalled; its answer is collected + /// and dropped by the loop, which no longer waits for it. + fn abandon(&mut self, ticket: SearchTicket) { + lock(&self.shared).searches.retain(|(queued, ..)| *queued != ticket); + } } /// Plain HTTP to one base address, in text: the adapters own what the text diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index e0e4ecf..6cce84f 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -12,7 +12,7 @@ use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, IndexStatus}; use huginn_mind::{Mind, OwnedRedbMessagePackBackingStore, PipelineAdmissionOutcome}; use tempfile::TempDir; -use super::fakes::{FakeEmbedder, FakeIndex, Gate, pair}; +use super::fakes::{FakeEmbedder, FakeIndex, Gate, Stored, pair}; use super::ollama::OllamaEmbedder; use super::qdrant::QdrantIndex; use super::*; @@ -615,9 +615,7 @@ fn dropping_the_sink_stops_the_worker() { sink.wait_status(|status| *status == IndexStatus::Current); let shared = Arc::downgrade(&sink.shared); drop(sink); - while shared.upgrade().is_some() { - std::thread::yield_now(); - } + eventually("the worker letting go of the shared state", || shared.upgrade().is_none()); } /// F1. The promised curve, pure: 30 s doubling to a 10 min cap, never below @@ -759,7 +757,7 @@ fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { /// Waits for a condition the worker will make true, without a clock in the /// assertion: it polls for at most ten seconds and then says what never /// happened. -fn eventually(what: &str, condition: impl Fn() -> bool) { +fn eventually(what: &str, mut condition: impl FnMut() -> bool) { for _ in 0..2000 { if condition() { return; @@ -864,7 +862,7 @@ fn a_search_refuses_when_the_index_is_not_reconciled_with_the_model() { let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - assert!(format!("{:#}", fresh.search("anything", 3).unwrap_err()).contains("not been reconciled")); + assert!(format!("{:#}", fresh.search("anything", 3).unwrap_err()).contains("being rebuilt")); assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); let mut projector = projected(&embedder, &index, &entries); @@ -931,41 +929,45 @@ fn shared_with( } /// The idle wait is for work: an entry, the mind's identity, a search or the -/// sink closing ends it at once, and only a quiet worker waits out the recheck -/// interval, after which it says the interval passed. +/// sink closing ends it at once, and only a quiet worker waits out its +/// deadline. The deadline is the caller's: a deadline already past ends the +/// wait at once too, and waking for work does not extend it. #[test] -fn an_idle_wait_ends_for_work_and_otherwise_at_the_recheck() { +fn an_idle_wait_ends_for_work_and_otherwise_at_its_deadline() { let (_root, mind) = mind_with_documents(); let entry = mind.index_entries(None).unwrap().remove(0); let waits = |shared: &Lock| { let started = Instant::now(); - let recheck = wait_for_work(shared, Wait::Work { recheck: Duration::from_secs(30) }); - (recheck, started.elapsed() < Duration::from_secs(10)) + wait_for_work(shared, Wait::Work { until: Instant::now() + Duration::from_secs(30) }); + started.elapsed() < Duration::from_secs(10) }; - assert_eq!(waits(&shared_with(vec![entry], None, Vec::new(), false)), (false, true), "an entry"); - assert_eq!(waits(&shared_with(Vec::new(), Some(MIND.into()), Vec::new(), false)), (false, true), "the mind's identity"); - assert_eq!(waits(&shared_with(Vec::new(), None, vec![(SearchTicket(0), "q".into(), 3)], false)), (false, true), "a search"); - assert_eq!(waits(&shared_with(Vec::new(), None, Vec::new(), true)), (false, true), "the sink closing"); + assert!(waits(&shared_with(vec![entry], None, Vec::new(), false)), "an entry"); + assert!(waits(&shared_with(Vec::new(), Some(MIND.into()), Vec::new(), false)), "the mind's identity"); + assert!(waits(&shared_with(Vec::new(), None, vec![(SearchTicket(0), "q".into(), 3)], false)), "a search"); + assert!(waits(&shared_with(Vec::new(), None, Vec::new(), true)), "the sink closing"); let quiet = shared_with(Vec::new(), None, Vec::new(), false); let started = Instant::now(); - assert!(wait_for_work(&quiet, Wait::Work { recheck: Duration::from_millis(60) }), "a quiet worker is told the interval passed"); - assert!(started.elapsed() >= Duration::from_millis(60), "and only after it did"); + wait_for_work(&quiet, Wait::Work { until: started + Duration::from_millis(60) }); + assert!(started.elapsed() >= Duration::from_millis(60), "a quiet worker waits for its deadline"); + let started = Instant::now(); + wait_for_work(&quiet, Wait::Work { until: started.checked_sub(Duration::from_secs(1)).unwrap_or(started) }); + assert!(started.elapsed() < Duration::from_secs(10), "a deadline already past is not waited for"); } -/// A backoff is the whole delay, whatever arrives, and never reports a -/// recheck: a failing index is retried on its schedule. +/// A backoff is the whole delay, whatever arrives: a failing index is retried +/// on its schedule. #[test] fn a_backoff_wait_is_the_whole_delay_whatever_arrives() { let (_root, mind) = mind_with_documents(); let entry = mind.index_entries(None).unwrap().remove(0); let busy = shared_with(vec![entry], Some(MIND.into()), vec![(SearchTicket(0), "q".into(), 3)], false); let started = Instant::now(); - assert!(!wait_for_work(&busy, Wait::Backoff(Duration::from_millis(80)))); + wait_for_work(&busy, Wait::Backoff(Duration::from_millis(80))); assert!(started.elapsed() >= Duration::from_millis(80), "the delay was waited out although work was queued"); let closed = shared_with(Vec::new(), None, Vec::new(), true); let started = Instant::now(); - assert!(!wait_for_work(&closed, Wait::Backoff(Duration::from_secs(30)))); + wait_for_work(&closed, Wait::Backoff(Duration::from_secs(30))); assert!(started.elapsed() < Duration::from_secs(10), "a closed sink ends even a backoff"); } @@ -986,3 +988,400 @@ fn queued_searches_are_answered_with_why_when_the_worker_gives_up() { ] ); } + +// The semantic read's index side: when a search is answered, how many wait, +// and what the worker does between them. + +type Answers = Vec<(SearchTicket, Result)>; + +/// The sink's own calls, named: the trait is generic over the store and the +/// worker does not care which. +fn ask_sink(sink: &mut WorkerSink, text: &str, top_k: u32) -> Result { + >::search(sink, text, top_k) +} + +fn collected(sink: &mut WorkerSink) -> Answers { + >::searched(sink) +} + +fn abandon(sink: &mut WorkerSink, ticket: SearchTicket) { + >::abandon(sink, ticket); +} + +/// Waits for `count` answers and returns them, in the order they were given. +fn answers(sink: &mut WorkerSink, count: usize) -> Answers { + let mut all = Vec::new(); + eventually("the searches being answered", || { + all.extend(collected(sink)); + all.len() >= count + }); + all +} + +/// `count` documents that share one document's text and differ in id, for the +/// tests whose subject is how many there are. +fn entries_of(count: usize) -> Vec { + let (_root, mind) = mind_with_documents(); + let template = mind.index_entries(None).unwrap().remove(0); + (0..count) + .map(|n| { + let mut entry = template.clone(); + entry.id.id.0 = format!("{}#{n}", entry.id.id.0); + entry + }) + .collect() +} + +fn queries(embedder: &FakeEmbedder) -> Vec { + embedder.texts().into_iter().filter(|text| text.starts_with("Instruct: ")).collect() +} + +/// Only a current index answers. Being reconciled, being written to, failing +/// and being refused each refuse by name, and the count of documents pending +/// is in the words. +#[test] +fn only_a_current_index_answers_a_search() { + use IndexStatus as S; + assert_eq!(unsearchable(&S::Current), None); + let says = |status: S, wanted: &[&str]| { + let why = unsearchable(&status).unwrap_or_else(|| panic!("{status:?} answered")); + for word in wanted { + assert!(why.contains(word), "{status:?}: {word} in {why}"); + } + }; + says(S::Reconciling { pending: 7 }, &["being rebuilt", "reconciling", "pending: 7"]); + says(S::Behind { pending: 5 }, &["being rebuilt", "behind", "pending: 5"]); + says(S::Failing { pending: 3, attempts: 2, error: "boom".into() }, &["failing", "boom", "pending: 3"]); + says(S::Refused { pending: 4, attempts: 2, reason: "not ours".into() }, &["refused", "not ours", "pending: 4"]); + + // The projector never answers from a half-filled collection, and never + // touches the embedder or the vector store to find out. + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + projector.want(entries[3..].iter().cloned()); + let embedded = embedder.texts().len(); + let refusal = format!("{:#}", projector.search("anything", 3).unwrap_err()); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 1"), "{refusal}"); + assert_eq!(embedder.texts().len(), embedded); + assert!(index.state.lock().unwrap().searches.is_empty()); + drain(&mut projector); + assert!(projector.search("anything", 3).is_ok(), "once it has caught up it answers"); + + // The sink refuses before the worker is asked, while the worker is inside a batch. + let gate = Gate::shut(); + let mut sink = WorkerSink::spawn( + FakeEmbedder::new("d1").behind(&gate), + FakeIndex::default(), + &slug(INSTANCE), + Some(MIND.into()), + entries.clone(), + quick(), + ); + gate.wait_arrived(1); + let refusal = ask_sink(&mut sink, "anything", 3).unwrap_err(); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); + gate.open(); + sink.wait_status(|status| *status == IndexStatus::Current); + assert!(ask_sink(&mut sink, "anything", 3).is_ok()); +} + +/// A search that fails after it was accepted has found the index not what it +/// claimed: the reconciliation is forgotten, so the status stops reading +/// `Current` and the worker verifies the model and the collection again. +#[test] +fn a_search_that_fails_forgets_the_reconciliation() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let breaks: [(&str, Box); 2] = [ + ("the vector store fails the search", Box::new(|_, index| index.state.lock().unwrap().fail_search = true)), + ("the embedder is down", Box::new(|embedder, _| embedder.state.lock().unwrap().down = true)), + ]; + for (what, breaks) in breaks { + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + assert_eq!(projector.progress_status(), IndexStatus::Current); + breaks(&embedder, &index); + assert!(projector.search("anything", 3).is_err(), "{what}"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 0 }, "{what}"); + assert!(projector.stale(), "{what}"); + } + + // In the worker the next step is the verification, with nothing to prompt it. + let (embedder, index) = pair(); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let listed = index.state.lock().unwrap().listed; + index.state.lock().unwrap().fail_search = true; + let ticket = ask_sink(&mut sink, "anything", 3).unwrap(); + let failed = answers(&mut sink, 1); + assert_eq!(failed[0].0, ticket); + assert!(failed[0].1.is_err()); + eventually("the worker listing the collection again", || index.state.lock().unwrap().listed > listed); +} + +/// A search asks the store what the collection says about itself: a label for +/// another mind, no label, or no collection is refused before the query is +/// embedded, and the reconciliation is to be redone. +#[test] +fn a_search_re_reads_the_collections_label() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let cases: [(&str, Box); 3] = [ + ( + "another mind's label", + Box::new(|stored| { + let Described::Labelled(meta) = &mut stored.label else { panic!() }; + meta.mind = "mind-commit-another".into(); + }), + ), + ("no label", Box::new(|stored| stored.label = Described::Unlabelled)), + ("another text version", Box::new(|stored| { + let Described::Labelled(meta) = &mut stored.label else { panic!() }; + meta.index_text_version += 1; + })), + ]; + for (what, change) in cases { + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + change(index.state.lock().unwrap().collections.get_mut(&collection()).unwrap()); + let embedded = embedder.texts().len(); + let refusal = format!("{:#}", projector.search("anything", 3).unwrap_err()); + assert!(refusal.contains("no longer the one this mind's index was built as"), "{what}: {refusal}"); + assert_eq!(embedder.texts().len(), embedded, "{what}: nothing was embedded"); + assert!(index.state.lock().unwrap().searches.is_empty(), "{what}"); + assert!(projector.stale(), "{what}"); + } + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + index.state.lock().unwrap().collections.clear(); + assert!(projector.search("anything", 3).is_err(), "no collection"); + assert!(projector.stale()); +} + +/// The model is read again after the query is embedded: a vector made while the +/// model changed is never compared with the collection. +#[test] +fn a_model_that_changes_while_the_query_is_embedded_is_refused() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let index = FakeIndex::default(); + gate.open(); + let mut projector = projected(&embedder, &index, &entries); + gate.shut_again(); + let searching = std::thread::spawn(move || { + let found = projector.search("anything", 3); + (projector, found) + }); + gate.wait_arrived(1); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + gate.open(); + let (projector, found) = searching.join().unwrap(); + let refusal = format!("{:#}", found.unwrap_err()); + assert!(refusal.contains("while the query was embedded"), "{refusal}"); + assert!(index.state.lock().unwrap().searches.is_empty(), "the vector was not used"); + assert!(projector.stale()); +} + +/// Each search has a ticket of its own, and its answer comes back once, as the +/// worker gave it: nothing is added to it and nothing is left to be +/// collected twice. +#[test] +fn every_search_has_its_own_ticket_and_its_answer_comes_back_exactly_once() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + index.state.lock().unwrap().hits = + vec![Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }]; + let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let expected: Hits = vec![( + PipelineRef { kind: PipelineKind::Question, id: Short("eureka-state:question:Q1".into()) }, + 0.9, + )]; + let mut tickets = Vec::new(); + for text in ["a", "b", "c"] { + let ticket = ask_sink(&mut sink, text, 3).unwrap(); + assert_eq!(answers(&mut sink, 1), vec![(ticket, Ok(expected.clone()))]); + assert!(collected(&mut sink).is_empty(), "an answer is handed over once"); + tickets.push(ticket); + } + assert_eq!(tickets, [SearchTicket(0), SearchTicket(1), SearchTicket(2)]); +} + +/// The queue of searches is bounded and the asker who does not fit is told. +/// One search is inside the embedder and the queue is full; the next is +/// refused by name, and when the worker has answered some it takes more. +#[test] +fn the_search_queue_is_bounded_and_the_asker_is_told_when_it_is_full() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + ask_sink(&mut sink, "in flight", 3).unwrap(); + gate.wait_arrived(1); + for n in 0..SEARCHES_QUEUED_MAX { + ask_sink(&mut sink, &format!("waiting {n}"), 3).unwrap_or_else(|why| panic!("{n} of {SEARCHES_QUEUED_MAX}: {why}")); + } + let refusal = ask_sink(&mut sink, "one too many", 3).unwrap_err(); + assert!(refusal.contains(&format!("{SEARCHES_QUEUED_MAX} searches waiting")), "{refusal}"); + + gate.open(); + let answered = answers(&mut sink, SEARCHES_QUEUED_MAX + 1); + assert_eq!(answered.len(), SEARCHES_QUEUED_MAX + 1); + assert!(ask_sink(&mut sink, "room again", 3).is_ok()); +} + +/// A search the asker has stopped waiting for is taken out of the queue and +/// never embedded; the ones on either side of it are. +#[test] +fn an_abandoned_search_is_never_embedded() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let first = ask_sink(&mut sink, "first", 3).unwrap(); + gate.wait_arrived(1); + let second = ask_sink(&mut sink, "second", 3).unwrap(); + let third = ask_sink(&mut sink, "third", 3).unwrap(); + abandon(&mut sink, second); + gate.open(); + let answered = answers(&mut sink, 2); + let tickets: Vec = answered.iter().map(|(ticket, _)| *ticket).collect(); + assert_eq!(tickets, [first, third]); + let texts = queries(&embedder); + assert_eq!(texts.len(), 2, "{texts:?}"); + assert!(texts[0].ends_with("Query: first") && texts[1].ends_with("Query: third"), "{texts:?}"); +} + +/// The worker serves one queued search and returns to its writes: three +/// waiting searches leave two after one turn. +#[test] +fn one_queued_search_is_served_per_turn() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + let queued = vec![(SearchTicket(1), "a".into(), 3), (SearchTicket(2), "b".into(), 3), (SearchTicket(3), "c".into(), 3)]; + let shared = shared_with(Vec::new(), None, queued, false); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert_eq!(guard.searches.iter().map(|(ticket, ..)| *ticket).collect::>(), [SearchTicket(2), SearchTicket(3)]); + assert_eq!(guard.found.iter().map(|(ticket, _)| *ticket).collect::>(), [SearchTicket(1)]); +} + +/// Searches are answered between batches, not after the last: a search waiting +/// while the first of two batches is written is answered, refused for the +/// documents still pending, before the second batch is embedded. +#[test] +fn a_waiting_search_is_answered_between_batches() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), quick()); + gate.wait_arrived(1); + lock(&sink.shared).searches.push_back((SearchTicket(9), "anything".into(), 3)); + gate.open(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, SearchTicket(9)); + let Err(refusal) = &answered[0].1 else { panic!("a search was answered from a half-filled collection: {answered:?}") }; + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 8"), "{refusal}"); + sink.wait_status(|status| *status == IndexStatus::Current); +} + +/// A search that was queued when the worker failed is answered with why, by +/// the worker's own failure path. +#[test] +fn a_search_queued_before_the_worker_fails_is_answered_with_why() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries_of(1), quick()); + gate.wait_arrived(1); + lock(&sink.shared).searches.push_back((SearchTicket(5), "anything".into(), 3)); + embedder.state.lock().unwrap().down = true; + gate.open(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, SearchTicket(5)); + let Err(why) = &answered[0].1 else { panic!("{answered:?}") }; + assert!(why.contains("the index could not advance") && why.contains("the embedder is down"), "{why}"); +} + +/// While a failure is being reported, the report's pending count is the one +/// the worker holds now, published under the lock that took the entries out of +/// the inbox: never the count from before they moved. +#[test] +fn a_failure_report_carries_the_pending_count_it_has_now() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); + projector.want(entries[..2].iter().cloned()); + let shared = shared_with(entries[2..].to_vec(), None, Vec::new(), false); + let error = StepError::Unavailable(anyhow::anyhow!("down")); + assert!(absorb(&mut projector, &shared, Some((&error, 3)))); + { + let guard = lock(&shared); + assert!(guard.inbox.is_empty()); + assert_eq!(guard.status, IndexStatus::Failing { pending: 4, attempts: 3, error: "down".into() }); + } + assert!(absorb(&mut projector, &shared, None)); + assert_eq!(lock(&shared).status, IndexStatus::Reconciling { pending: 4 }); + lock(&shared).closed = true; + assert!(!absorb(&mut projector, &shared, None), "a closed sink ends the worker"); +} + +/// The worker's recheck is on its own clock: searches arriving faster than the +/// interval do not keep pushing it out. +#[test] +fn steady_searches_do_not_postpone_the_workers_recheck() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let backoff = Backoff { recheck: Duration::from_millis(200), ..quick() }; + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); + sink.wait_status(|status| *status == IndexStatus::Current); + let listed = index.state.lock().unwrap().listed; + let started = Instant::now(); + while index.state.lock().unwrap().listed < listed + 2 { + assert!(started.elapsed() < Duration::from_secs(10), "the worker never rechecked while searches kept arriving"); + let _ = ask_sink(&mut sink, "anything", 3); + collected(&mut sink); + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// `wait_status` gives up at one deadline, not one per wake: a health that +/// keeps changing does not keep it waiting. +#[test] +fn wait_status_gives_up_at_its_overall_deadline_however_often_the_health_changes() { + let (embedder, index) = pair(); + let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let shared = Arc::clone(&sink.shared); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let ticking = Arc::clone(&stop); + let ticker = std::thread::spawn(move || { + while !ticking.load(std::sync::atomic::Ordering::Relaxed) { + shared.1.notify_all(); + std::thread::sleep(Duration::from_millis(5)); + } + }); + let (tell, told) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let gave_up = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + sink.wait_status_for(Duration::from_millis(300), |status| *status == IndexStatus::Reconciling { pending: 99 }) + })) + .is_err(); + let _ = tell.send(gave_up); + }); + let gave_up = told.recv_timeout(Duration::from_secs(10)); + stop.store(true, std::sync::atomic::Ordering::Relaxed); + ticker.join().unwrap(); + assert_eq!(gave_up, Ok(true), "the wait did not end at its deadline"); +} diff --git a/crates/huginn-daemon/src/semantic_tests.rs b/crates/huginn-daemon/src/semantic_tests.rs index ad8bb19..406f066 100644 --- a/crates/huginn-daemon/src/semantic_tests.rs +++ b/crates/huginn-daemon/src/semantic_tests.rs @@ -297,10 +297,10 @@ fn a_semantic_query_returns_only_documents_the_mind_holds_and_the_selection_admi assert!(searches.iter().all(|(name, _, limit)| *name == collection_name(&slug(INSTANCE)) && *limit == 20), "{searches:?}"); } -/// A cursor cannot resume a ranked answer: it is refused by name, and nothing -/// is embedded or searched for it. +/// A cursor cannot resume a ranked answer and a ranked answer has no order to +/// reverse: each is refused by name, and nothing is embedded or searched for it. #[test] -fn a_cursor_with_a_semantic_query_is_refused_and_asks_the_index_nothing() { +fn a_cursor_or_a_reversal_with_a_semantic_query_is_refused_and_asks_the_index_nothing() { let harness = healthy(); let mut client = harness.client(); until_index(&mut client, "Current", |status| *status == IndexStatus::Current); @@ -314,6 +314,15 @@ fn a_cursor_with_a_semantic_query_is_refused_and_asks_the_index_nothing() { ); assert_eq!(harness.embedder.texts().len(), embedded); assert!(harness.index.state.lock().unwrap().searches.is_empty()); + + let reversed = Selection { descending: true, ..standing() }; + let refusal = ask(&mut client, "m-2", &semantic(reversed, 5)); + assert!( + matches!(&refusal, HuginnMindResponse::Refused(MindRefusal::SelectionInvalid { field, .. }) if field == "descending"), + "{refusal:?}" + ); + assert_eq!(harness.embedder.texts().len(), embedded); + assert!(harness.index.state.lock().unwrap().searches.is_empty()); } /// The loop never waits on the network. The embedder is shut, the worker is @@ -412,3 +421,59 @@ fn an_index_that_cannot_answer_is_unavailable_and_plain_queries_still_work() { assert!(detail.contains("refused") && detail.contains("no Huginn metadata"), "{detail}"); assert_eq!(documents(&mut client, "m-plain-3"), held); } + +/// A search the client has stopped waiting for is not started. The embedder is +/// held with the first search inside it; a second search queues behind it and +/// times out, and when the embedder is released the worker embeds the first, +/// skips the second and embeds a third: the index is never asked for an answer +/// nobody is waiting for. +#[test] +fn a_search_that_timed_out_while_queued_is_never_embedded() { + let gate = Gate::shut(); + gate.open(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let harness = Harness::start(shelf(), embedder, FakeIndex::default(), Duration::from_millis(300)); + let mut first = harness.client(); + let mut second = harness.client(); + let mut third = harness.client(); + until_index(&mut third, "Current", |status| *status == IndexStatus::Current); + let queries = |harness: &Harness| harness.embedder.texts().iter().filter(|text| text.starts_with("Instruct: ")).count(); + + gate.shut_again(); + send(&mut first, "m-first", &semantic(standing(), 5)); + gate.wait_arrived(1); + send(&mut second, "m-second", &semantic(standing(), 5)); + let timed_out = receive(&mut second, "m-second", 3000).expect("the second search is answered at its deadline"); + assert!(matches!(&timed_out, HuginnMindResponse::Refused(MindRefusal::Unavailable { detail }) if detail.contains("did not finish")), "{timed_out:?}"); + + gate.open(); + send(&mut third, "m-third", &semantic(standing(), 5)); + let answered = receive(&mut third, "m-third", 3000).expect("the third search is answered"); + assert!(matches!(answered, HuginnMindResponse::Query(_)), "{answered:?}"); + assert_eq!(queries(&harness), 2, "the first and the third were embedded; the second was taken back before it started"); +} + +/// While the index is rebuilding, a semantic query is refused with an +/// `Unavailable` that names the state and how many documents are pending, and +/// the embedder is not asked for it. When the index has caught up the same +/// query is answered. +#[test] +fn a_search_while_the_index_is_being_rebuilt_is_refused_and_asks_the_embedder_nothing() { + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let harness = Harness::start(shelf(), embedder, FakeIndex::default(), Duration::from_secs(30)); + gate.wait_arrived(1); + let mut client = harness.client(); + let IndexStatus::Behind { pending } = index_status(&mut client, "m-status") else { panic!("the worker is inside its first batch") }; + assert!(pending > 0); + + let refused = ask(&mut client, "m-behind", &semantic(standing(), 5)); + let HuginnMindResponse::Refused(MindRefusal::Unavailable { detail }) = &refused else { panic!("{refused:?}") }; + assert!(detail.contains("being rebuilt") && detail.contains(&format!("pending: {pending}")), "{detail}"); + assert!(harness.embedder.texts().is_empty(), "nothing was embedded, not even the documents, for a refused search"); + + gate.open(); + until_index(&mut client, "Current", |status| *status == IndexStatus::Current); + harness.index.state.lock().unwrap().hits = vec![hit("ruling", id("ruling", "R4"), 0.5)]; + assert_eq!(ids(&ask(&mut client, "m-current", &semantic(standing(), 5))), vec![id("ruling", "R4")]); +} diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index 593ae63..628abb9 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -519,7 +519,8 @@ pub fn run>( } /// What the index has finished, then what has waited too long: the latter as -/// an `Unavailable` detail. A search that finishes after its deadline finds +/// an `Unavailable` detail, and abandoned, so the index does not start a search +/// nobody is waiting for. A search that finishes after its deadline finds /// nothing waiting for it and is dropped. fn collect_searches>( daemon: &mut Daemon, @@ -530,6 +531,7 @@ fn collect_searches>( let now = Instant::now(); for (ticket, asked) in waiting { if asked.until <= now { + daemon.abandon(*ticket); collected.push((*ticket, Err(format!("the semantic search did not finish within {:?}", options.search_timeout)))); } } From 2052d314e347cbe8879baaa0a39b1c6a1cbcaee4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 16:54:18 +0200 Subject: [PATCH 015/520] Pin what the mutant run left open; make failing tests fail fast cargo-mutants on 11b's diff: three timeouts (a drain loop that never settled under Projector::reconcile -> Ok(()), a dead worker leaving thirty-second waits to add up past the harness deadline) and three survivors. Now: - drain is bounded, the status and gate waits are ten seconds, so a mutant fails inside the harness deadline instead of hanging it; - the worker's first step is due immediately rather than one interval after a startup that is itself a verification, and an idle worker is pinned to one recheck per interval (a recheck that did not move its own deadline would list the collection in a loop); - the log line for hits the mind does not hold is a function with a test. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/daemon.rs | 36 ++++++++++++++++++++----- crates/huginn-daemon/src/index/fakes.rs | 2 +- crates/huginn-daemon/src/index/mod.rs | 6 +++-- crates/huginn-daemon/src/index/tests.rs | 28 ++++++++++++++++--- 4 files changed, 59 insertions(+), 13 deletions(-) diff --git a/crates/huginn-daemon/src/daemon.rs b/crates/huginn-daemon/src/daemon.rs index c42d644..8b831a3 100644 --- a/crates/huginn-daemon/src/daemon.rs +++ b/crates/huginn-daemon/src/daemon.rs @@ -70,6 +70,20 @@ impl Handled { } } +/// What the log says of the hits the mind held no document for: how many, and +/// the first few by name (a restored mind can leave hundreds), or nothing when +/// the mind held them all. +fn dropped_note(dropped: &[PipelineRef]) -> Option { + if dropped.is_empty() { + return None; + } + let named: Vec<&str> = dropped.iter().take(5).map(|hit| hit.id.0.as_str()).collect(); + Some(format!( + "huginn: the index returned {} ids this mind does not hold (first {named:?}); they are left out", + dropped.len() + )) +} + /// The envelope's `source_runtime_id` on every response a daemon serving this /// instance sends. Also what the socket is bound under, before a daemon exists. pub fn runtime_id(instance: &Slug) -> String { @@ -181,13 +195,8 @@ impl> Daemon { }; match self.mind.rank(&search.selection, &search.semantic, &hits) { Ok(ranked) => { - if !ranked.dropped.is_empty() { - let named: Vec<&str> = ranked.dropped.iter().take(5).map(|hit| hit.id.0.as_str()).collect(); - eprintln!( - "huginn: the index returned {} ids this mind does not hold (first {:?}); they are left out", - ranked.dropped.len(), - named - ); + if let Some(note) = dropped_note(&ranked.dropped) { + eprintln!("{note}"); } HuginnMindResponse::Query(ranked.page) } @@ -428,6 +437,19 @@ pub(crate) mod tests { (root, daemon) } + /// The log names how many hits the mind did not hold and the first five, and + /// says nothing when it held them all. + #[test] + fn the_dropped_hits_are_reported_by_count_and_the_first_five_names() { + assert_eq!(dropped_note(&[]), None); + let dropped: Vec = + (0..7).map(|n| PipelineRef { kind: PipelineKind::Question, id: Short(format!("gone-{n}")) }).collect(); + let note = dropped_note(&dropped).unwrap(); + assert!(note.contains("returned 7 ids"), "{note}"); + assert!(note.contains("gone-0") && note.contains("gone-4") && !note.contains("gone-5"), "{note}"); + assert_eq!(dropped_note(&dropped[..1]).unwrap().matches("gone-").count(), 1); + } + /// No index: for the tests whose subject is not the projection. pub(crate) struct NoIndex; diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs index 3ed7560..ffb8570 100644 --- a/crates/huginn-daemon/src/index/fakes.rs +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -24,7 +24,7 @@ impl Gate { /// Blocks until `count` calls to `embed` have arrived at the gate. pub(crate) fn wait_arrived(&self, count: u32) { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); let mut state = self.0.0.lock().unwrap(); while state.0 < count { let left = deadline.saturating_duration_since(std::time::Instant::now()); diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs index 1d7aaa6..c0980c8 100644 --- a/crates/huginn-daemon/src/index/mod.rs +++ b/crates/huginn-daemon/src/index/mod.rs @@ -699,7 +699,9 @@ fn work(mut projector: Projector, shared: Loc // When the worker next verifies the model and the collection whether or // not anything has woken it: an absolute time, moved only by a // verification, so a steady stream of searches cannot keep pushing it out. - let mut recheck_at = Instant::now() + backoff.recheck; + // The first step is a verification (the startup reconciliation), so it is + // due now. + let mut recheck_at = Instant::now(); loop { if !absorb(&mut projector, &shared, failing.as_ref().map(|(error, attempts)| (error, *attempts))) { return; @@ -811,7 +813,7 @@ impl WorkerSink { } /// The longest `WorkerSink::wait_status` waits, in all. -const WAIT_STATUS_MAX: Duration = Duration::from_secs(30); +const WAIT_STATUS_MAX: Duration = Duration::from_secs(10); impl Drop for WorkerSink { fn drop(&mut self) { diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index 6cce84f..2e7387c 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -59,8 +59,15 @@ fn counts(index: &FakeIndex) -> (usize, usize) { (state.recreated.len(), state.upserts.len()) } +/// Steps until the projector has nothing to do. A step that never settles is a +/// failure of the test, not a hang: it is bounded. fn drain(projector: &mut Projector) { - while projector.advance().unwrap() == Advance::Progressed {} + for _ in 0..1000 { + if projector.advance().unwrap() == Advance::Idle { + return; + } + } + panic!("the projector was still stepping after 1000 steps"); } fn point_ids(entries: &[IndexEntry]) -> BTreeSet { @@ -758,7 +765,7 @@ fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { /// assertion: it polls for at most ten seconds and then says what never /// happened. fn eventually(what: &str, mut condition: impl FnMut() -> bool) { - for _ in 0..2000 { + for _ in 0..1000 { if condition() { return; } @@ -1349,7 +1356,7 @@ fn steady_searches_do_not_postpone_the_workers_recheck() { let listed = index.state.lock().unwrap().listed; let started = Instant::now(); while index.state.lock().unwrap().listed < listed + 2 { - assert!(started.elapsed() < Duration::from_secs(10), "the worker never rechecked while searches kept arriving"); + assert!(started.elapsed() < Duration::from_secs(5), "the worker never rechecked while searches kept arriving"); let _ = ask_sink(&mut sink, "anything", 3); collected(&mut sink); std::thread::sleep(Duration::from_millis(10)); @@ -1385,3 +1392,18 @@ fn wait_status_gives_up_at_its_overall_deadline_however_often_the_health_changes ticker.join().unwrap(); assert_eq!(gave_up, Ok(true), "the wait did not end at its deadline"); } + +/// An idle worker verifies once per interval, not in a loop: the recheck moves +/// its own deadline when it runs. +#[test] +fn an_idle_worker_rechecks_once_per_interval() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let backoff = Backoff { recheck: Duration::from_millis(100), ..quick() }; + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); + sink.wait_status(|status| *status == IndexStatus::Current); + std::thread::sleep(Duration::from_millis(600)); + let listed = index.state.lock().unwrap().listed; + assert!((2..=40).contains(&listed), "the startup reconciliation and about five rechecks listed the collection {listed} times"); +} From de2344da95128f85a921a1ec840b2fe54be5586d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:16:03 +0200 Subject: [PATCH 016/520] Name the queries in the abandonment test The three searches now carry their own text, so the test asserts which two were embedded, in what order, not only how many. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/semantic_tests.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/crates/huginn-daemon/src/semantic_tests.rs b/crates/huginn-daemon/src/semantic_tests.rs index 406f066..6880009 100644 --- a/crates/huginn-daemon/src/semantic_tests.rs +++ b/crates/huginn-daemon/src/semantic_tests.rs @@ -110,10 +110,14 @@ fn standing() -> Selection { } fn semantic(selection: Selection, top_k: u32) -> HuginnMindRequest { + semantic_of("who owns the state", selection, top_k) +} + +fn semantic_of(text: &str, selection: Selection, top_k: u32) -> HuginnMindRequest { HuginnMindRequest::Query { instance: slug(INSTANCE), selection, - semantic: Some(SemanticQuery { text: "who owns the state".into(), top_k }), + semantic: Some(SemanticQuery { text: text.into(), top_k }), } } @@ -437,20 +441,24 @@ fn a_search_that_timed_out_while_queued_is_never_embedded() { let mut second = harness.client(); let mut third = harness.client(); until_index(&mut third, "Current", |status| *status == IndexStatus::Current); - let queries = |harness: &Harness| harness.embedder.texts().iter().filter(|text| text.starts_with("Instruct: ")).count(); + let queries = |harness: &Harness| -> Vec { + harness.embedder.texts().into_iter().filter(|text| text.starts_with("Instruct: ")).collect() + }; gate.shut_again(); - send(&mut first, "m-first", &semantic(standing(), 5)); + send(&mut first, "m-first", &semantic_of("first", standing(), 5)); gate.wait_arrived(1); - send(&mut second, "m-second", &semantic(standing(), 5)); + send(&mut second, "m-second", &semantic_of("second", standing(), 5)); let timed_out = receive(&mut second, "m-second", 3000).expect("the second search is answered at its deadline"); assert!(matches!(&timed_out, HuginnMindResponse::Refused(MindRefusal::Unavailable { detail }) if detail.contains("did not finish")), "{timed_out:?}"); gate.open(); - send(&mut third, "m-third", &semantic(standing(), 5)); + send(&mut third, "m-third", &semantic_of("third", standing(), 5)); let answered = receive(&mut third, "m-third", 3000).expect("the third search is answered"); assert!(matches!(answered, HuginnMindResponse::Query(_)), "{answered:?}"); - assert_eq!(queries(&harness), 2, "the first and the third were embedded; the second was taken back before it started"); + let embedded = queries(&harness); + assert_eq!(embedded.len(), 2, "the first and the third were embedded; the second was taken back before it started: {embedded:?}"); + assert!(embedded[0].ends_with("Query: first") && embedded[1].ends_with("Query: third"), "{embedded:?}"); } /// While the index is rebuilding, a semantic query is refused with an From 01925b829fa8ec908465bec88ea250698b717acd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:50:34 +0200 Subject: [PATCH 017/520] Bound the query embed by the search deadline; one check-then-embed for search and flush Embedder::embed takes the bound it must answer within. The query embed gets QUERY_EMBED_TIMEOUT, derived from SEARCH_DEADLINE, which is also the serve loop's default search timeout. The bulk flush embed keeps 300 s (BULK_EMBED_TIMEOUT); it does not block serving. Projector::embed_verified reads the model identity and the collection label, embeds, re-reads the identity and refuses on change. Search and flush both use it; the inline copies and step()'s separate identity read are gone. Flush now also refuses a collection that is no longer this mind's. Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/src/index/fakes.rs | 13 ++- crates/huginn-daemon/src/index/mod.rs | 81 +++++++++++------ crates/huginn-daemon/src/index/ollama.rs | 21 +++-- crates/huginn-daemon/src/index/tests.rs | 110 +++++++++++++++++++++-- crates/huginn-daemon/src/serve.rs | 2 +- 5 files changed, 182 insertions(+), 45 deletions(-) diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs index ffb8570..47502b8 100644 --- a/crates/huginn-daemon/src/index/fakes.rs +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; use anyhow::{Result, bail, ensure}; @@ -60,6 +61,11 @@ pub(crate) struct EmbedderState { pub(crate) down: bool, /// The texts of every completed call, in order. pub(crate) embedded: Vec>, + /// The bound each completed call was given, in order. + pub(crate) bounds: Vec, + /// The length of the vectors it answers with, where that is not the + /// identity's. + pub(crate) vector_length: Option, } #[derive(Clone)] @@ -70,7 +76,7 @@ pub(crate) struct FakeEmbedder { impl FakeEmbedder { pub(crate) fn new(digest: &str) -> Self { - let state = EmbedderState { identity: identity(digest), down: false, embedded: Vec::new() }; + let state = EmbedderState { identity: identity(digest), down: false, embedded: Vec::new(), bounds: Vec::new(), vector_length: None }; Self { state: Arc::new(Mutex::new(state)), gate: None } } @@ -93,7 +99,7 @@ impl Embedder for FakeEmbedder { Ok(state.identity.clone()) } - fn embed(&mut self, texts: &[String]) -> Result>> { + fn embed(&mut self, texts: &[String], within: Duration) -> Result>> { if let Some(gate) = &self.gate { gate.pass(); } @@ -102,7 +108,8 @@ impl Embedder for FakeEmbedder { bail!("the embedder is down"); } state.embedded.push(texts.to_vec()); - let dimensions = state.identity.dimensions as usize; + state.bounds.push(within); + let dimensions = state.vector_length.unwrap_or(state.identity.dimensions as usize); Ok(texts.iter().map(|text| (0..dimensions).map(|at| (text.len() + at) as f32).collect()).collect()) } } diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs index c0980c8..3f9102c 100644 --- a/crates/huginn-daemon/src/index/mod.rs +++ b/crates/huginn-daemon/src/index/mod.rs @@ -65,7 +65,8 @@ //! (`ServeOptions::search_timeout` bounds the wait), and takes back a search //! it has stopped waiting for. At most `SEARCHES_QUEUED_MAX` wait; one more is //! refused. A search whose embed is already running cannot be recalled, so it -//! can hold the writes off for as long as Ollama's own timeout. +//! can hold the writes off for as long as its bound, `QUERY_EMBED_TIMEOUT`, +//! which is `SEARCH_DEADLINE`, the deadline the serve loop gives the search. //! - Only a current index answers a search. While the worker is `Reconciling` //! or `Behind` (a re-pulled model, a lost collection, entries not yet //! written), `Failing` or `Refused`, a search is refused at once with an @@ -99,6 +100,22 @@ pub const MANAGED_BY: &str = "huginn"; /// Documents embedded and written per call. pub const BATCH: usize = 32; +/// How long a search may take, queue wait included: the serve loop's default +/// `ServeOptions::search_timeout`. It is the one number the query embed's +/// bound derives from, so the two cannot drift apart. +pub const SEARCH_DEADLINE: Duration = Duration::from_secs(30); + +/// The bound on one query embed. A running embed cannot be recalled, so it +/// holds the index's writes off for as long as it lasts; it is held to the +/// search's own deadline, after which nobody is waiting for its vector. +pub const QUERY_EMBED_TIMEOUT: Duration = SEARCH_DEADLINE; + +/// The bound on a bulk index-flush embed, and on the embedder's identity +/// reads. A cold model takes seconds to load. It does not block serving: the +/// serve loop never waits on the worker, and a search queued behind a flush is +/// answered by its own deadline. +pub const BULK_EMBED_TIMEOUT: Duration = Duration::from_secs(300); + /// The most candidates one search asks the index for. pub const OVERSAMPLE_MAX: u32 = 200; @@ -163,8 +180,8 @@ pub struct Hit { pub trait Embedder { fn model_identity(&mut self) -> Result; - /// One vector per text, in order. - fn embed(&mut self, texts: &[String]) -> Result>>; + /// One vector per text, in order, or an error once `within` has passed. + fn embed(&mut self, texts: &[String], within: Duration) -> Result>>; } pub trait VectorIndex { @@ -319,9 +336,6 @@ impl Projector { if self.wanted.is_empty() { return Ok(Advance::Idle); } - if !self.model_unchanged()? { - return Ok(Advance::Progressed); - } self.flush_batch()?; Ok(Advance::Progressed) } @@ -408,7 +422,9 @@ impl Projector { return Ok(()); } let texts: Vec = batch.iter().map(|entry| entry.text.clone()).collect(); - let vectors = self.embedder.embed(&texts).context("embedding")?; + // A model that changed leaves nothing written and the reconciliation + // forgotten: the next step reconciles and rebuilds. + let Ok(vectors) = self.embed_verified(&texts, BULK_EMBED_TIMEOUT)? else { return Ok(()) }; ensure!(vectors.len() == batch.len(), "the embedder returned {} vectors for {} texts", vectors.len(), batch.len()); let mut points = Vec::with_capacity(batch.len()); for (entry, vector) in batch.iter().zip(vectors) { @@ -437,6 +453,33 @@ impl Projector { Ok(()) } + /// The one check-then-embed: reads the model's identity and the + /// collection's label, embeds, reads the identity again, and gives the + /// vectors back only if the model is still the one the collection was + /// reconciled under. A model that changed forgets the reconciliation and + /// answers `Err(when)`; a collection that is no longer this mind's, built + /// as reconciled, is an error and forgets it too. Search and flush both + /// embed through here, so neither compares or writes vectors made by a + /// model the collection cannot mix. + fn embed_verified(&mut self, texts: &[String], within: Duration) -> Result>, &'static str>> { + let (Some(mind), Some(reconciled)) = (self.mind.clone(), self.reconciled.clone()) else { + bail!("the mind has no identity yet, so it has no index"); + }; + if !self.model_unchanged().map_err(|error| anyhow::anyhow!("{error}"))? { + return Ok(Err("since the collection was built")); + } + let described = self.index.describe(&self.collection).context("describing the collection")?; + if described != Described::Labelled(self.collection_meta(&mind, &reconciled)) { + self.reconciled = None; + bail!("collection {} is no longer the one this mind's index was built as: {described:?}", self.collection); + } + let vectors = self.embedder.embed(texts, within).context("embedding")?; + if !self.model_unchanged().map_err(|error| anyhow::anyhow!("{error}"))? { + return Ok(Err("while the texts were embedded")); + } + Ok(Ok(vectors)) + } + /// The candidates a query text finds: the text is embedded with the model's /// query instruction and the collection is searched, with no filter and an /// oversample of `min(top_k * 4, 200)`, because the mind will drop most of @@ -465,31 +508,17 @@ impl Projector { } fn nearest(&mut self, text: &str, top_k: u32) -> Result { - let (Some(mind), Some(reconciled)) = (self.mind.clone(), self.reconciled.clone()) else { + let Some(reconciled) = self.reconciled.clone() else { bail!("the mind has no identity yet, so it has no index"); }; let dimensions = reconciled.dimensions as usize; - ensure!( - self.model_unchanged().map_err(|error| anyhow::anyhow!("{error}"))?, - "the embedding model changed since the collection was built; the index is being rebuilt for it" - ); - let described = self.index.describe(&self.collection).context("describing the collection")?; - ensure!( - described == Described::Labelled(self.collection_meta(&mind, &reconciled)), - "collection {} is no longer the one this mind's index was built as: {described:?}", - self.collection - ); let query = format!("Instruct: {QUERY_INSTRUCTION}\nQuery: {text}"); - let mut vectors = self.embedder.embed(&[query]).context("embedding the query")?; + let mut vectors = match self.embed_verified(&[query], QUERY_EMBED_TIMEOUT)? { + Ok(vectors) => vectors, + Err(when) => bail!("the embedding model changed {when}; the index is being rebuilt for it"), + }; ensure!(vectors.len() == 1, "the embedder returned {} vectors for one query", vectors.len()); let vector = vectors.remove(0); - // The model is read again after the query was embedded: only a vector - // made between two reads of the same model is compared with the - // collection. - ensure!( - self.model_unchanged().map_err(|error| anyhow::anyhow!("{error}"))?, - "the embedding model changed while the query was embedded; the index is being rebuilt for it" - ); ensure!(vector.len() == dimensions, "the embedder returned a query vector of {} where the collection holds {dimensions}", vector.len()); let found = self.index.search(&self.collection, &vector, top_k.saturating_mul(4).min(OVERSAMPLE_MAX))?; let mut hits = Vec::with_capacity(found.len()); diff --git a/crates/huginn-daemon/src/index/ollama.rs b/crates/huginn-daemon/src/index/ollama.rs index 850b276..603bd08 100644 --- a/crates/huginn-daemon/src/index/ollama.rs +++ b/crates/huginn-daemon/src/index/ollama.rs @@ -6,7 +6,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail, ensure}; use serde_json::{Value, json}; -use super::{Embedder, Http, Method, ModelIdentity}; +use super::{BULK_EMBED_TIMEOUT, Embedder, Http, Method, ModelIdentity}; /// The instruction Qwen3 embedding models take in front of a query (never in /// front of a document). Cut 11b's semantic read prepends it as @@ -15,20 +15,22 @@ pub const QUERY_INSTRUCTION: &str = "Given a question about a project's pipeline state, retrieve the documents that answer it"; /// One model on one Ollama. A cold model takes seconds to load, so the -/// timeout is generous; the worker owns the schedule around it. +/// identity reads get `BULK_EMBED_TIMEOUT`; the worker owns the schedule +/// around it. Each embed call carries its own bound. pub struct OllamaEmbedder { http: Http, + base_url: String, model: String, } impl OllamaEmbedder { pub fn new(base_url: &str, model: &str) -> Self { - Self { http: Http::new(base_url, Duration::from_secs(300)), model: model.to_owned() } + Self { http: Http::new(base_url, BULK_EMBED_TIMEOUT), base_url: base_url.to_owned(), model: model.to_owned() } } - fn ask(&self, method: Method, path: &str, body: Option<&Value>) -> Result { + fn ask(&self, http: &Http, method: Method, path: &str, body: Option<&Value>) -> Result { let text = body.map(Value::to_string); - let (status, answer) = self.http.request(method, path, text.as_deref())?; + let (status, answer) = http.request(method, path, text.as_deref())?; ensure!(status == 200, "Ollama answered {status} to {path}: {}", answer.chars().take(300).collect::()); serde_json::from_str(&answer).with_context(|| format!("Ollama's answer to {path} is not JSON")) } @@ -38,7 +40,7 @@ impl Embedder for OllamaEmbedder { /// The configured name, the digest Ollama lists for it, and the model's /// embedding length. The address is not part of the identity. fn model_identity(&mut self) -> Result { - let tags = self.ask(Method::Get, "/api/tags", None)?; + let tags = self.ask(&self.http, Method::Get, "/api/tags", None)?; let listed = tags["models"].as_array().context("Ollama's tag list has no models")?; let digest = listed .iter() @@ -46,7 +48,7 @@ impl Embedder for OllamaEmbedder { .and_then(|entry| entry["digest"].as_str()) .with_context(|| format!("Ollama does not list the model {}", self.model))? .to_owned(); - let shown = self.ask(Method::Post, "/api/show", Some(&json!({ "model": self.model })))?; + let shown = self.ask(&self.http, Method::Post, "/api/show", Some(&json!({ "model": self.model })))?; let info = shown["model_info"].as_object().context("Ollama's model description has no model_info")?; let Some(length) = info.iter().find(|(key, _)| key.ends_with(".embedding_length")).and_then(|(_, value)| value.as_u64()) else { @@ -56,8 +58,9 @@ impl Embedder for OllamaEmbedder { Ok(ModelIdentity { name: self.model.clone(), digest, dimensions }) } - fn embed(&mut self, texts: &[String]) -> Result>> { - let answer = self.ask(Method::Post, "/api/embed", Some(&json!({ "model": self.model, "input": texts })))?; + fn embed(&mut self, texts: &[String], within: Duration) -> Result>> { + let bounded = Http::new(&self.base_url, within); + let answer = self.ask(&bounded, Method::Post, "/api/embed", Some(&json!({ "model": self.model, "input": texts })))?; let embeddings = answer["embeddings"].as_array().context("Ollama's answer has no embeddings")?; embeddings .iter() diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index 2e7387c..c36699b 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -235,8 +235,8 @@ fn a_collection_owned_by_another_instance_is_never_recreated() { } } -/// A vector of the wrong length is refused before anything is written, and the -/// documents stay wanted. +/// The vector's length is checked against the collection's, so a flush that +/// the model did not change is still refused when the embedder answers wrongly. #[test] fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { let (_root, mind) = mind_with_documents(); @@ -245,13 +245,111 @@ fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); projector.want(entries.iter().cloned()); projector.reconcile().unwrap(); - embedder.state.lock().unwrap().identity.dimensions = 8; + embedder.state.lock().unwrap().vector_length = Some(8); let error = projector.flush_batch().expect_err("a longer vector is refused"); assert!(matches!(error, StepError::Unavailable(_)), "{error}"); assert_eq!(projector.pending(), 4); assert!(index.state.lock().unwrap().upserts.is_empty()); } +/// A flush embeds through the same check as a search: a model that changed +/// before the embed is never asked, a model that changed during it has its +/// vectors dropped, and a collection that is no longer this mind's is refused. +/// In each case nothing is written, the documents stay wanted, and the +/// reconciliation is forgotten. +#[test] +fn a_flush_is_refused_as_a_search_is_when_the_model_or_the_collection_changed() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let reconciled = || { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + projector.reconcile().unwrap(); + (embedder, index, projector) + }; + + let (embedder, index, mut projector) = reconciled(); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + projector.flush_batch().expect("a changed model is not a failure"); + assert!(embedder.texts().is_empty(), "before: the texts were not embedded under the new model"); + assert!(index.state.lock().unwrap().upserts.is_empty()); + assert_eq!((projector.pending(), projector.stale()), (4, true), "before"); + + let (embedder, index, mut projector) = reconciled(); + change_label(&index, |meta| meta.mind = "mind-commit-another".into()); + let error = projector.flush_batch().expect_err("another mind's label is refused"); + assert!(error.to_string().contains("no longer the one this mind's index was built as"), "{error}"); + assert!(embedder.texts().is_empty(), "label: nothing was embedded"); + assert!(index.state.lock().unwrap().upserts.is_empty()); + assert_eq!((projector.pending(), projector.stale()), (4, true), "label"); + + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let index = FakeIndex::default(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + projector.reconcile().unwrap(); + let flushing = std::thread::spawn(move || { + let flushed = projector.flush_batch(); + (projector, flushed) + }); + gate.wait_arrived(1); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + gate.open(); + let (projector, flushed) = flushing.join().unwrap(); + flushed.expect("a changed model is not a failure"); + assert!(index.state.lock().unwrap().upserts.is_empty(), "during: the vectors were not written"); + assert_eq!((projector.pending(), projector.stale()), (4, true), "during"); +} + +fn change_label(index: &FakeIndex, change: impl FnOnce(&mut CollectionMeta)) { + let mut state = index.state.lock().unwrap(); + let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; + change(meta); +} + +/// The query embed is bounded by the search's deadline, one constant behind +/// both, so a query embed that cannot be recalled cannot hold the writes off +/// past the moment nobody is waiting for it. A bulk flush keeps its own bound. +#[test] +fn the_query_embed_is_bounded_by_the_search_deadline_and_the_flush_by_its_own() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + assert!(!embedder.state.lock().unwrap().bounds.is_empty()); + assert!(embedder.state.lock().unwrap().bounds.iter().all(|bound| *bound == BULK_EMBED_TIMEOUT), "the flushes"); + projector.search("anything", 3).unwrap(); + let bounds = embedder.state.lock().unwrap().bounds.clone(); + assert_eq!(bounds.last(), Some(&QUERY_EMBED_TIMEOUT), "the query"); + let served = crate::serve::ServeOptions::default().search_timeout; + assert_eq!(served, SEARCH_DEADLINE); + assert!(QUERY_EMBED_TIMEOUT <= served); + assert!(QUERY_EMBED_TIMEOUT < BULK_EMBED_TIMEOUT); +} + +/// The Ollama adapter enforces the bound it is given: an embed that is never +/// answered fails at the bound, not at the adapter's default. +#[test] +fn an_ollama_embed_that_is_never_answered_fails_at_its_bound() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let held = std::thread::spawn(move || { + let mut held = Vec::new(); + for stream in listener.incoming().take(1) { + held.push(stream.unwrap()); + std::thread::sleep(Duration::from_secs(8)); + } + }); + let (sent, received) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut embedder = OllamaEmbedder::new(&url, "m:1"); + sent.send(embedder.embed(&["a".into()], Duration::from_millis(300)).is_err()).ok(); + }); + assert_eq!(received.recv_timeout(Duration::from_secs(5)), Ok(true), "the embed did not fail at its bound"); + drop(held); +} fn whoami_index(daemon: &mut Daemon) -> IndexStatus { match daemon.handle(HuginnMindRequest::Whoami, now()).answered() { HuginnMindResponse::Whoami(status) => status.index, @@ -446,7 +544,7 @@ fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { let ollama = stub(&[("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]); let mut embedder = OllamaEmbedder::new(&ollama.url, "m:1"); assert_eq!(embedder.model_identity().unwrap(), ModelIdentity { name: "m:1".into(), digest: "ac6d".into(), dimensions: 4 }); - let vectors = embedder.embed(&["a".into(), "b".into(), "c".into(), "d".into()]).unwrap(); + let vectors = embedder.embed(&["a".into(), "b".into(), "c".into(), "d".into()], Duration::from_secs(5)).unwrap(); assert_eq!(vectors.len(), 4); assert_eq!(vectors[0], vec![0.5, 1.0, 2.0, 3.0]); let seen = ollama.seen.lock().unwrap(); @@ -464,7 +562,7 @@ fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { let mut absent = OllamaEmbedder::new(&ollama.url, "not-pulled:1"); assert!(format!("{:#}", absent.model_identity().unwrap_err()).contains("does not list the model")); let refusing = stub(&[("POST /api/embed", 500, r#"{"error":"boom"}"#)]); - assert!(OllamaEmbedder::new(&refusing.url, "m:1").embed(&["a".into()]).is_err()); + assert!(OllamaEmbedder::new(&refusing.url, "m:1").embed(&["a".into()], Duration::from_secs(5)).is_err()); } /// Compatibility never reads the address: the same model at another address @@ -1188,7 +1286,7 @@ fn a_model_that_changes_while_the_query_is_embedded_is_refused() { gate.open(); let (projector, found) = searching.join().unwrap(); let refusal = format!("{:#}", found.unwrap_err()); - assert!(refusal.contains("while the query was embedded"), "{refusal}"); + assert!(refusal.contains("while the texts were embedded"), "{refusal}"); assert!(index.state.lock().unwrap().searches.is_empty(), "the vector was not used"); assert!(projector.stale()); } diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index 628abb9..5af8589 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -100,7 +100,7 @@ impl Default for ServeOptions { session_timeout: Duration::from_secs(30), idle_sleep: Duration::from_millis(2), deferred_ttl: Duration::from_secs(60), - search_timeout: Duration::from_secs(30), + search_timeout: crate::index::SEARCH_DEADLINE, } } } From e899c65774c8a8db95a32490207d1d45280ac09a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:53:24 +0200 Subject: [PATCH 018/520] Give the live index smoke the embed bound Co-Authored-By: Claude Sonnet 5 --- crates/huginn-daemon/tests/live_index_smoke.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/huginn-daemon/tests/live_index_smoke.rs b/crates/huginn-daemon/tests/live_index_smoke.rs index dbb6f49..7f6b4d4 100644 --- a/crates/huginn-daemon/tests/live_index_smoke.rs +++ b/crates/huginn-daemon/tests/live_index_smoke.rs @@ -10,6 +10,7 @@ //! cargo test -p huginn-daemon --test live_index_smoke -- --ignored --nocapture use std::collections::BTreeSet; +use std::time::Duration; use huginn_daemon::index::ollama::{OllamaEmbedder, QUERY_INSTRUCTION}; use huginn_daemon::index::qdrant::QdrantIndex; @@ -46,7 +47,7 @@ fn the_adapters_embed_write_query_and_clean_up_a_scratch_collection() { let identity = embedder.model_identity().expect("the model is listed"); println!("identity: {identity:?}"); let texts = ["The organ owns the mind.".to_string(), "A recipe for lentil soup.".to_string()]; - let vectors = embedder.embed(&texts).expect("two texts embed"); + let vectors = embedder.embed(&texts, Duration::from_secs(60)).expect("two texts embed"); assert_eq!(vectors.len(), 2); assert!(vectors.iter().all(|vector| vector.len() == identity.dimensions as usize)); @@ -95,7 +96,7 @@ fn the_adapters_embed_write_query_and_clean_up_a_scratch_collection() { // instruction the worker uses. let asked = format!("Instruct: {QUERY_INSTRUCTION} Query: Who is responsible for keeping the memory?"); - let query = embedder.embed(&[asked]).expect("the query embeds").remove(0); + let query = embedder.embed(&[asked], Duration::from_secs(60)).expect("the query embeds").remove(0); let hits = store.search(name, &query, 2).unwrap(); assert_eq!(hits.len(), 2, "{hits:?}"); assert_eq!((hits[0].doc_id.as_str(), hits[0].kind.as_str()), ("smoke:doc:0", "question"), "{hits:?}"); From 4a4d8b029ee62ff97c0cc0246e6f86aade3d54c3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:42:03 +0200 Subject: [PATCH 019/520] Bound the whole search path by the search's own time; lag is not a rebuild Soul pass 2 follow-ups F1, F2, F4, F5 on the semantic index. F1: every call a search makes (identity reads, label read, embed, query) is bounded by what remains of one Budget, and the queued ticket carries the moment SEARCH_DEADLINE after it was asked. The ports take the bound per call (model_identity, describe, search); QUERY_EMBED_TIMEOUT is gone, since the ticket's deadline is the one number. A hung identity read frees the worker within the search's time. F2: one refusal predicate. Behind is lag and no longer refuses; only Reconciling, Failing and Refused do, for the sink and the projector alike. The worker absorbs the inbox before taking a search and leaves the search queued while the writes ahead of it are unfinished, so a client that admits and then searches reads its own writes. An index being rebuilt is still refused. F4: the model-changed refusal and the worker's failure answer to queued searches name the state and the pending count. F5: a search that fails for its own reasons no longer forgets the reconciliation; only a changed model or label does. Co-Authored-By: Claude Opus 5.5 --- crates/huginn-daemon/src/index/fakes.rs | 31 +- crates/huginn-daemon/src/index/mod.rs | 226 +++++++----- crates/huginn-daemon/src/index/ollama.rs | 16 +- crates/huginn-daemon/src/index/qdrant.rs | 21 +- crates/huginn-daemon/src/index/tests.rs | 324 +++++++++++++----- crates/huginn-daemon/src/semantic_tests.rs | 26 +- .../huginn-daemon/tests/live_index_smoke.rs | 8 +- 7 files changed, 436 insertions(+), 216 deletions(-) diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs index 47502b8..61c98a2 100644 --- a/crates/huginn-daemon/src/index/fakes.rs +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -66,6 +66,11 @@ pub(crate) struct EmbedderState { /// The length of the vectors it answers with, where that is not the /// identity's. pub(crate) vector_length: Option, + /// The bound each identity read was given, in order. + pub(crate) identity_bounds: Vec, + /// Whether an identity read is never answered: it fails when its bound + /// has passed, as an adapter that honours its bound does. + pub(crate) hang_identity: bool, } #[derive(Clone)] @@ -76,7 +81,7 @@ pub(crate) struct FakeEmbedder { impl FakeEmbedder { pub(crate) fn new(digest: &str) -> Self { - let state = EmbedderState { identity: identity(digest), down: false, embedded: Vec::new(), bounds: Vec::new(), vector_length: None }; + let state = EmbedderState { identity: identity(digest), down: false, embedded: Vec::new(), bounds: Vec::new(), vector_length: None, identity_bounds: Vec::new(), hang_identity: false }; Self { state: Arc::new(Mutex::new(state)), gate: None } } @@ -91,8 +96,14 @@ impl FakeEmbedder { } impl Embedder for FakeEmbedder { - fn model_identity(&mut self) -> Result { - let state = self.state.lock().unwrap(); + fn model_identity(&mut self, within: Duration) -> Result { + let mut state = self.state.lock().unwrap(); + state.identity_bounds.push(within); + if state.hang_identity { + drop(state); + std::thread::sleep(within); + bail!("the embedder did not answer within {within:?}"); + } if state.down { bail!("the embedder is down"); } @@ -135,6 +146,10 @@ pub(crate) struct IndexState { pub(crate) listed: u32, /// Whether a search fails at the vector store, the rest of it working. pub(crate) fail_search: bool, + /// The bound each describe and each search was given, in order, by call. + pub(crate) bounds: Vec<(&'static str, Duration)>, + /// How many points the collection held when each search reached it. + pub(crate) held_when_searched: Vec, } #[derive(Clone, Default)] @@ -155,8 +170,9 @@ impl FakeIndex { } impl VectorIndex for FakeIndex { - fn describe(&mut self, collection: &str) -> Result { - let state = self.state.lock().unwrap(); + fn describe(&mut self, collection: &str, within: Duration) -> Result { + let mut state = self.state.lock().unwrap(); + state.bounds.push(("describe", within)); if state.down { bail!("the vector store is down"); } @@ -197,14 +213,17 @@ impl VectorIndex for FakeIndex { Ok(()) } - fn search(&mut self, collection: &str, vector: &[f32], limit: u32) -> Result> { + fn search(&mut self, collection: &str, vector: &[f32], limit: u32, within: Duration) -> Result> { let mut state = self.state.lock().unwrap(); + state.bounds.push(("search", within)); if state.down { bail!("the vector store is down"); } state.searches.push((collection.into(), vector.len(), limit)); ensure!(!state.fail_search, "the vector store failed the search"); ensure!(state.collections.contains_key(collection), "no collection {collection}"); + let held = state.collections[collection].points.len(); + state.held_when_searched.push(held); Ok(state.hits.clone()) } } diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs index 3f9102c..367e675 100644 --- a/crates/huginn-daemon/src/index/mod.rs +++ b/crates/huginn-daemon/src/index/mod.rs @@ -65,15 +65,24 @@ //! (`ServeOptions::search_timeout` bounds the wait), and takes back a search //! it has stopped waiting for. At most `SEARCHES_QUEUED_MAX` wait; one more is //! refused. A search whose embed is already running cannot be recalled, so it -//! can hold the writes off for as long as its bound, `QUERY_EMBED_TIMEOUT`, -//! which is `SEARCH_DEADLINE`, the deadline the serve loop gives the search. -//! - Only a current index answers a search. While the worker is `Reconciling` -//! or `Behind` (a re-pulled model, a lost collection, entries not yet -//! written), `Failing` or `Refused`, a search is refused at once with an -//! `Unavailable` that names the state and how many documents are pending; it -//! is never answered from a half-filled collection. A search that fails after -//! it was accepted forgets the reconciliation, so `whoami` stops reporting -//! `Current` and the worker verifies the model and the collection again. +//! can hold the writes off for as long as it has left: the ticket carries the +//! moment `SEARCH_DEADLINE` after it was asked, and every call the search +//! makes (the model's identity, the collection's label, the embed, the +//! query) is bounded by what remains of it. +//! - A search reads its own writes. Admissions and searches meet under one +//! lock, admissions first: a search waits in the queue while the worker +//! absorbs and writes what was admitted before it, and is answered once the +//! index has caught up. Lag is not a rebuild and refuses nothing. Under a +//! steady stream of admissions a search may never see the index caught up; +//! its deadline then answers it. +//! - A search is refused only while the index is being rebuilt or cannot be +//! asked: `Reconciling` (a re-pulled model, a lost collection, a mind's first +//! admission), `Failing` or `Refused`. The refusal is an `Unavailable` that +//! names the state and how many documents are pending; it is never answered +//! from a collection being rebuilt. A search that fails for its own reasons +//! (a timeout, a store error) fails that search alone and the +//! reconciliation stands. Only what the check-then-embed finds changed (the +//! model, the collection's label) forgets it. pub mod ollama; pub mod qdrant; @@ -101,21 +110,21 @@ pub const MANAGED_BY: &str = "huginn"; pub const BATCH: usize = 32; /// How long a search may take, queue wait included: the serve loop's default -/// `ServeOptions::search_timeout`. It is the one number the query embed's -/// bound derives from, so the two cannot drift apart. +/// `ServeOptions::search_timeout`. Every bound on the search path derives from +/// the moment this long after the search was asked, so the two cannot drift +/// apart. pub const SEARCH_DEADLINE: Duration = Duration::from_secs(30); -/// The bound on one query embed. A running embed cannot be recalled, so it -/// holds the index's writes off for as long as it lasts; it is held to the -/// search's own deadline, after which nobody is waiting for its vector. -pub const QUERY_EMBED_TIMEOUT: Duration = SEARCH_DEADLINE; - -/// The bound on a bulk index-flush embed, and on the embedder's identity -/// reads. A cold model takes seconds to load. It does not block serving: the -/// serve loop never waits on the worker, and a search queued behind a flush is -/// answered by its own deadline. +/// The bound on a bulk index-flush embed, the identity and label reads around +/// it, and the reconciliation's identity read. A cold model takes seconds to +/// load. It does not block serving: the serve loop never waits on the worker, +/// and a search queued behind a flush is answered by its own deadline. pub const BULK_EMBED_TIMEOUT: Duration = Duration::from_secs(300); +/// The bound on a call to the vector store that is not a search: the +/// adapter's default. +pub const INDEX_CALL_TIMEOUT: Duration = Duration::from_secs(60); + /// The most candidates one search asks the index for. pub const OVERSAMPLE_MAX: u32 = 200; @@ -179,13 +188,16 @@ pub struct Hit { } pub trait Embedder { - fn model_identity(&mut self) -> Result; + /// The model's identity, or an error once `within` has passed. + fn model_identity(&mut self, within: Duration) -> Result; /// One vector per text, in order, or an error once `within` has passed. fn embed(&mut self, texts: &[String], within: Duration) -> Result>>; } pub trait VectorIndex { - fn describe(&mut self, collection: &str) -> Result; + /// What the collection says about itself, or an error once `within` has + /// passed. + fn describe(&mut self, collection: &str, within: Duration) -> Result; /// Drops the collection if it exists and creates it empty with this /// metadata. fn recreate(&mut self, collection: &str, meta: &CollectionMeta) -> Result<()>; @@ -193,8 +205,9 @@ pub trait VectorIndex { fn ids(&mut self, collection: &str) -> Result>; fn upsert(&mut self, collection: &str, points: &[Point]) -> Result<()>; /// The `limit` nearest points to `vector`, best first, with no filter: which - /// documents may be shown is the mind's to decide. - fn search(&mut self, collection: &str, vector: &[f32], limit: u32) -> Result>; + /// documents may be shown is the mind's to decide. An error once `within` + /// has passed. + fn search(&mut self, collection: &str, vector: &[f32], limit: u32, within: Duration) -> Result>; } /// The mind's collection: one per instance. @@ -245,6 +258,25 @@ pub enum Advance { Idle, } +/// The time one operation has left. Every call the operation makes is bounded +/// by what remains, not by a bound of its own, so their sum cannot outlast it. +struct Budget { + end: Instant, + total: Duration, +} + +impl Budget { + fn new(total: Duration) -> Self { + Self { end: Instant::now() + total, total } + } + + fn left(&self) -> Result { + let left = self.end.saturating_duration_since(Instant::now()); + ensure!(!left.is_zero(), "the time allowed ({:?}) has run out", self.total); + Ok(left) + } +} + /// The projection's state: the entries not yet known to be in the collection, /// and, once reconciled, the length of vector it accepts. pub struct Projector { @@ -344,8 +376,8 @@ impl Projector { /// collection was reconciled under. A different model, digest or length /// forgets the reconciliation, so the next step reconciles (and rebuilds) /// instead of writing or searching with vectors the collection cannot mix. - fn model_unchanged(&mut self) -> Result { - let now = self.embedder.model_identity().context("re-reading the embedding model's identity")?; + fn model_unchanged(&mut self, budget: &Budget) -> Result { + let now = self.embedder.model_identity(budget.left()?).context("re-reading the embedding model's identity")?; if self.reconciled.as_ref() == Some(&now) { return Ok(true); } @@ -362,9 +394,10 @@ impl Projector { let Some(mind) = self.mind.clone() else { return Err(StepError::Unavailable(anyhow::anyhow!("the mind has no identity yet"))); }; - let identity = self.embedder.model_identity().context("reading the embedding model's identity")?; + let identity = + self.embedder.model_identity(BULK_EMBED_TIMEOUT).context("reading the embedding model's identity")?; let wanted = self.collection_meta(&mind, &identity); - match self.index.describe(&self.collection).context("describing the collection")? { + match self.index.describe(&self.collection, INDEX_CALL_TIMEOUT).context("describing the collection")? { Described::Absent => self.recreate(&wanted)?, Described::Unlabelled => { return Err(StepError::Refused(format!( @@ -411,10 +444,10 @@ impl Projector { /// the first `BATCH` wanted entries are embedded, written, and only then /// no longer wanted. pub fn flush_batch(&mut self) -> Result<(), StepError> { - self.flush().map_err(StepError::Unavailable) + self.flush(&Budget::new(BULK_EMBED_TIMEOUT)).map_err(StepError::Unavailable) } - fn flush(&mut self) -> Result<()> { + fn flush(&mut self, budget: &Budget) -> Result<()> { let dimensions = self.reconciled.as_ref().context("flush before reconcile")?.dimensions as usize; let batch: Vec = self.wanted.iter().take(BATCH).filter_map(|document_id| self.known.get(document_id)).cloned().collect(); @@ -424,7 +457,7 @@ impl Projector { let texts: Vec = batch.iter().map(|entry| entry.text.clone()).collect(); // A model that changed leaves nothing written and the reconciliation // forgotten: the next step reconciles and rebuilds. - let Ok(vectors) = self.embed_verified(&texts, BULK_EMBED_TIMEOUT)? else { return Ok(()) }; + let Ok(vectors) = self.embed_verified(&texts, budget)? else { return Ok(()) }; ensure!(vectors.len() == batch.len(), "the embedder returned {} vectors for {} texts", vectors.len(), batch.len()); let mut points = Vec::with_capacity(batch.len()); for (entry, vector) in batch.iter().zip(vectors) { @@ -453,28 +486,29 @@ impl Projector { Ok(()) } - /// The one check-then-embed: reads the model's identity and the - /// collection's label, embeds, reads the identity again, and gives the - /// vectors back only if the model is still the one the collection was - /// reconciled under. A model that changed forgets the reconciliation and - /// answers `Err(when)`; a collection that is no longer this mind's, built - /// as reconciled, is an error and forgets it too. Search and flush both - /// embed through here, so neither compares or writes vectors made by a - /// model the collection cannot mix. - fn embed_verified(&mut self, texts: &[String], within: Duration) -> Result>, &'static str>> { + /// The one check-then-embed, every call bounded by what `budget` has left: + /// reads the model's identity and the collection's label, embeds, reads the + /// identity again, and gives the vectors back only if the model is still + /// the one the collection was reconciled under. A model that changed + /// forgets the reconciliation and answers `Err(when)`; a collection that + /// is no longer this mind's, built as reconciled, is an error and forgets + /// it too. A call that fails or times out forgets nothing. Search and + /// flush both embed through here, so neither compares or writes vectors + /// made by a model the collection cannot mix. + fn embed_verified(&mut self, texts: &[String], budget: &Budget) -> Result>, &'static str>> { let (Some(mind), Some(reconciled)) = (self.mind.clone(), self.reconciled.clone()) else { bail!("the mind has no identity yet, so it has no index"); }; - if !self.model_unchanged().map_err(|error| anyhow::anyhow!("{error}"))? { + if !self.model_unchanged(budget)? { return Ok(Err("since the collection was built")); } - let described = self.index.describe(&self.collection).context("describing the collection")?; + let described = self.index.describe(&self.collection, budget.left()?).context("describing the collection")?; if described != Described::Labelled(self.collection_meta(&mind, &reconciled)) { self.reconciled = None; bail!("collection {} is no longer the one this mind's index was built as: {described:?}", self.collection); } - let vectors = self.embedder.embed(texts, within).context("embedding")?; - if !self.model_unchanged().map_err(|error| anyhow::anyhow!("{error}"))? { + let vectors = self.embedder.embed(texts, budget.left()?).context("embedding")?; + if !self.model_unchanged(budget)? { return Ok(Err("while the texts were embedded")); } Ok(Ok(vectors)) @@ -486,41 +520,38 @@ impl Projector { /// what the index cannot know is not in force. A hit naming a kind this /// organ has no name for is logged and left out. /// - /// A search is answered only from an index that is current: while it is - /// being reconciled or written to, it refuses, naming why and how many - /// documents are pending, rather than answer from a half-filled - /// collection. It is also answered only from a collection that is still - /// this mind's, built by the model that is still the embedder: the model's - /// identity is read before and after the query is embedded and the - /// collection's label before it, and the query is refused if either - /// disagrees with what the collection was reconciled under. Any failure - /// after that forgets the reconciliation, so `progress_status` stops - /// reporting `Current` and the worker verifies again before it answers. - pub fn search(&mut self, text: &str, top_k: u32) -> Result { + /// A search is refused while the index is being rebuilt or cannot be asked + /// (`unsearchable`), naming the state and how many documents are pending. + /// Catching up on lag is the worker's: it serves a search only once the + /// documents admitted before it are written. The answer is also only from + /// a collection that is still this mind's, built by the model that is + /// still the embedder: the identity is read before and after the query is + /// embedded and the label before it, and the query is refused, with the + /// reconciliation forgotten, if either disagrees with what the collection + /// was reconciled under. Every call is bounded by what remains of + /// `within`. A search that fails for any other reason (a timeout, a store + /// error) fails alone and leaves the reconciliation standing. + pub fn search(&mut self, text: &str, top_k: u32, within: Duration) -> Result { if let Some(why) = unsearchable(&self.progress_status()) { bail!("{why}"); } - let found = self.nearest(text, top_k); - if found.is_err() { - self.reconciled = None; - } - found - } - - fn nearest(&mut self, text: &str, top_k: u32) -> Result { + let budget = Budget::new(within); let Some(reconciled) = self.reconciled.clone() else { bail!("the mind has no identity yet, so it has no index"); }; let dimensions = reconciled.dimensions as usize; let query = format!("Instruct: {QUERY_INSTRUCTION}\nQuery: {text}"); - let mut vectors = match self.embed_verified(&[query], QUERY_EMBED_TIMEOUT)? { + let mut vectors = match self.embed_verified(&[query], &budget)? { Ok(vectors) => vectors, - Err(when) => bail!("the embedding model changed {when}; the index is being rebuilt for it"), + Err(when) => { + bail!("the embedding model changed {when}: {}", unsearchable(&self.progress_status()).unwrap_or_default()) + } }; ensure!(vectors.len() == 1, "the embedder returned {} vectors for one query", vectors.len()); let vector = vectors.remove(0); ensure!(vector.len() == dimensions, "the embedder returned a query vector of {} where the collection holds {dimensions}", vector.len()); - let found = self.index.search(&self.collection, &vector, top_k.saturating_mul(4).min(OVERSAMPLE_MAX))?; + let limit = top_k.saturating_mul(4).min(OVERSAMPLE_MAX); + let found = self.index.search(&self.collection, &vector, limit, budget.left()?)?; let mut hits = Vec::with_capacity(found.len()); for hit in found { match PipelineKind::ALL.into_iter().find(|kind| kind.name() == hit.kind) { @@ -579,6 +610,16 @@ impl Backoff { /// told, not by the worker's patience. pub const SEARCHES_QUEUED_MAX: usize = 16; +/// A search waiting for the worker: what was asked, and the moment nobody is +/// waiting for the answer any more. Every call the search makes is bounded by +/// what remains until `due`. +struct Queued { + ticket: SearchTicket, + text: String, + top_k: u32, + due: Instant, +} + struct Shared { inbox: Vec, /// The mind's identity, once a commit has revealed it to a mind that had none. @@ -588,7 +629,7 @@ struct Shared { /// Searches started and not yet taken by the worker, at most /// `SEARCHES_QUEUED_MAX`; the serve loop takes back one it has stopped /// waiting for. - searches: VecDeque<(SearchTicket, String, u32)>, + searches: VecDeque, /// Searches the worker finished and the serve loop has not collected. found: Vec<(SearchTicket, Result)>, next_ticket: u64, @@ -655,18 +696,19 @@ fn including(status: &IndexStatus, queued: u32) -> IndexStatus { } } -/// Why a status cannot answer a search, or `None` when it can. Only a current -/// index answers: one that is being reconciled or written to holds a -/// half-filled collection, and one that is failing or refused cannot be asked. -/// The reason names the state and how many documents are pending. +/// Why a status cannot answer a search, or `None` when it can: the one +/// predicate for asking and for answering. An index being rebuilt +/// (`Reconciling`) holds a collection that is not the model's yet, and one +/// that is failing or refused cannot be asked. `Behind` is lag, not a +/// rebuild: the worker writes what was admitted before a search and then +/// answers it. The reason names the state and how many documents are pending. fn unsearchable(status: &IndexStatus) -> Option { use IndexStatus as S; match status { - S::Current => None, + S::Current | S::Behind { .. } => None, S::Reconciling { pending } => Some(format!( "the semantic index is being rebuilt: reconciling with the model and the collection (pending: {pending})" )), - S::Behind { pending } => Some(format!("the semantic index is being rebuilt: behind the mind (pending: {pending})")), S::Failing { pending, error, .. } => Some(format!("the semantic index is failing: {error} (pending: {pending})")), S::Refused { pending, reason, .. } => Some(format!("the semantic index is refused: {reason} (pending: {pending})")), } @@ -676,9 +718,18 @@ fn unsearchable(status: &IndexStatus) -> Option { /// for the loop to collect. Between batches, never inside one, and only one: /// the worker returns to its writes after each, so a queue of searches cannot /// hold the index's writes off for the sum of their embeddings. +/// +/// Admissions come first: the inbox is absorbed before the search is taken, so +/// every document committed before the search was asked is wanted, and a search +/// behind lag stays queued until the worker's writes have caught up. An index +/// being rebuilt does not wait: the search is taken and refused. The search's +/// calls are bounded by what remains of the time it was given when it was asked. fn serve_search(projector: &mut Projector, shared: &Lock) { - let Some((ticket, text, top_k)) = lock(shared).searches.pop_front() else { return }; - let found = projector.search(&text, top_k).map_err(|error| format!("{error:#}")); + if !absorb(projector, shared, None) || (projector.pending() > 0 && !projector.stale()) { + return; + } + let Some(Queued { ticket, text, top_k, due }) = lock(shared).searches.pop_front() else { return }; + let found = projector.search(&text, top_k, due.saturating_duration_since(Instant::now())).map_err(|error| format!("{error:#}")); lock(shared).found.push((ticket, found)); } @@ -686,8 +737,8 @@ fn serve_search(projector: &mut Projector, sh /// while it was healthy and it has failed since. fn refuse_searches(shared: &Lock, why: &str) { let mut guard = lock(shared); - while let Some((ticket, ..)) = guard.searches.pop_front() { - guard.found.push((ticket, Err(why.to_owned()))); + while let Some(queued) = guard.searches.pop_front() { + guard.found.push((queued.ticket, Err(why.to_owned()))); } } @@ -748,8 +799,8 @@ fn work(mut projector: Projector, shared: Loc } publish(&shared, projector.progress_status()); serve_search(&mut projector, &shared); - // A search that failed has left the reconciliation to redo: - // do it now, not at the next wake. + // A model or collection the search found changed has left the + // reconciliation to redo: do it now, not at the next wake. if advance == Advance::Idle && !projector.stale() { wait_for_work(&shared, Wait::Work { until: recheck_at }); } @@ -762,7 +813,7 @@ fn work(mut projector: Projector, shared: Loc delay ); publish(&shared, projector.failure_status(&error, attempts)); - refuse_searches(&shared, &format!("the index could not advance: {error}")); + refuse_searches(&shared, &unsearchable(&projector.failure_status(&error, attempts)).unwrap_or_default()); failing = Some((error, attempts)); wait_for_work(&shared, Wait::Backoff(delay)); delay = backoff.after(delay); @@ -875,8 +926,10 @@ impl IndexSink for WorkerSink { } /// Queues the search for the worker and returns its ticket. An index that - /// is not current is not asked, and neither is a full queue: the refusal - /// is the answer, naming why and how many documents are pending. + /// is being rebuilt or cannot be asked is not asked, and neither is a full + /// queue: the refusal is the answer, naming why and how many documents are + /// pending. Documents admitted and not yet written are not a refusal: the + /// search waits behind them. fn search(&mut self, text: &str, top_k: u32) -> Result { let mut guard = lock(&self.shared); let status = including(&guard.status, u32::try_from(guard.inbox.len()).unwrap_or(u32::MAX)); @@ -891,7 +944,7 @@ impl IndexSink for WorkerSink { } let ticket = SearchTicket(guard.next_ticket); guard.next_ticket += 1; - guard.searches.push_back((ticket, text.to_owned(), top_k)); + guard.searches.push_back(Queued { ticket, text: text.to_owned(), top_k, due: Instant::now() + SEARCH_DEADLINE }); drop(guard); self.shared.1.notify_all(); Ok(ticket) @@ -905,7 +958,7 @@ impl IndexSink for WorkerSink { /// the worker already holds cannot be recalled; its answer is collected /// and dropped by the loop, which no longer waits for it. fn abandon(&mut self, ticket: SearchTicket) { - lock(&self.shared).searches.retain(|(queued, ..)| *queued != ticket); + lock(&self.shared).searches.retain(|queued| queued.ticket != ticket); } } @@ -932,6 +985,11 @@ impl Http { Self { agent, base: base_url.trim_end_matches('/').to_owned() } } + /// The same address under a bound of its own. + pub(crate) fn bounded(&self, timeout: Duration) -> Self { + Self::new(&self.base, timeout) + } + /// The status and the body, whatever the status is. pub(crate) fn request(&self, method: Method, path: &str, body: Option<&str>) -> Result<(u16, String)> { let url = format!("{}{path}", self.base); diff --git a/crates/huginn-daemon/src/index/ollama.rs b/crates/huginn-daemon/src/index/ollama.rs index 603bd08..0461729 100644 --- a/crates/huginn-daemon/src/index/ollama.rs +++ b/crates/huginn-daemon/src/index/ollama.rs @@ -6,7 +6,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail, ensure}; use serde_json::{Value, json}; -use super::{BULK_EMBED_TIMEOUT, Embedder, Http, Method, ModelIdentity}; +use super::{Embedder, Http, Method, ModelIdentity}; /// The instruction Qwen3 embedding models take in front of a query (never in /// front of a document). Cut 11b's semantic read prepends it as @@ -14,18 +14,15 @@ use super::{BULK_EMBED_TIMEOUT, Embedder, Http, Method, ModelIdentity}; pub const QUERY_INSTRUCTION: &str = "Given a question about a project's pipeline state, retrieve the documents that answer it"; -/// One model on one Ollama. A cold model takes seconds to load, so the -/// identity reads get `BULK_EMBED_TIMEOUT`; the worker owns the schedule -/// around it. Each embed call carries its own bound. +/// One model on one Ollama. Every call carries its own bound. pub struct OllamaEmbedder { - http: Http, base_url: String, model: String, } impl OllamaEmbedder { pub fn new(base_url: &str, model: &str) -> Self { - Self { http: Http::new(base_url, BULK_EMBED_TIMEOUT), base_url: base_url.to_owned(), model: model.to_owned() } + Self { base_url: base_url.to_owned(), model: model.to_owned() } } fn ask(&self, http: &Http, method: Method, path: &str, body: Option<&Value>) -> Result { @@ -39,8 +36,9 @@ impl OllamaEmbedder { impl Embedder for OllamaEmbedder { /// The configured name, the digest Ollama lists for it, and the model's /// embedding length. The address is not part of the identity. - fn model_identity(&mut self) -> Result { - let tags = self.ask(&self.http, Method::Get, "/api/tags", None)?; + fn model_identity(&mut self, within: Duration) -> Result { + let http = Http::new(&self.base_url, within); + let tags = self.ask(&http, Method::Get, "/api/tags", None)?; let listed = tags["models"].as_array().context("Ollama's tag list has no models")?; let digest = listed .iter() @@ -48,7 +46,7 @@ impl Embedder for OllamaEmbedder { .and_then(|entry| entry["digest"].as_str()) .with_context(|| format!("Ollama does not list the model {}", self.model))? .to_owned(); - let shown = self.ask(&self.http, Method::Post, "/api/show", Some(&json!({ "model": self.model })))?; + let shown = self.ask(&http, Method::Post, "/api/show", Some(&json!({ "model": self.model })))?; let info = shown["model_info"].as_object().context("Ollama's model description has no model_info")?; let Some(length) = info.iter().find(|(key, _)| key.ends_with(".embedding_length")).and_then(|(_, value)| value.as_u64()) else { diff --git a/crates/huginn-daemon/src/index/qdrant.rs b/crates/huginn-daemon/src/index/qdrant.rs index 2999220..f4a1c85 100644 --- a/crates/huginn-daemon/src/index/qdrant.rs +++ b/crates/huginn-daemon/src/index/qdrant.rs @@ -7,7 +7,7 @@ use std::time::Duration; use anyhow::{Context, Result, ensure}; use serde_json::{Value, json}; -use super::{CollectionMeta, Described, Hit, Http, Method, Point, VectorIndex}; +use super::{CollectionMeta, INDEX_CALL_TIMEOUT, Described, Hit, Http, Method, Point, VectorIndex}; /// One Qdrant. Every path names a collection this organ was told to own; it /// never lists or touches another. @@ -20,12 +20,16 @@ const SCROLL_PAGE: u64 = 1000; impl QdrantIndex { pub fn new(base_url: &str) -> Self { - Self { http: Http::new(base_url, Duration::from_secs(60)) } + Self { http: Http::new(base_url, INDEX_CALL_TIMEOUT) } } fn ask(&self, method: Method, path: &str, body: Option<&Value>, ok: &[u16]) -> Result<(u16, Value)> { + self.ask_with(&self.http, method, path, body, ok) + } + + fn ask_with(&self, http: &Http, method: Method, path: &str, body: Option<&Value>, ok: &[u16]) -> Result<(u16, Value)> { let text = body.map(Value::to_string); - let (status, answer) = self.http.request(method, path, text.as_deref())?; + let (status, answer) = http.request(method, path, text.as_deref())?; ensure!(ok.contains(&status), "Qdrant answered {status} to {path}: {}", answer.chars().take(300).collect::()); let parsed = if answer.is_empty() { Value::Null } else { serde_json::from_str(&answer).unwrap_or(Value::Null) }; Ok((status, parsed)) @@ -67,8 +71,9 @@ fn described(metadata: &Value) -> Described { } impl VectorIndex for QdrantIndex { - fn describe(&mut self, collection: &str) -> Result { - let (status, answer) = self.ask(Method::Get, &format!("/collections/{collection}"), None, &[200, 404])?; + fn describe(&mut self, collection: &str, within: Duration) -> Result { + let http = self.http.bounded(within); + let (status, answer) = self.ask_with(&http, Method::Get, &format!("/collections/{collection}"), None, &[200, 404])?; if status == 404 { return Ok(Described::Absent); } @@ -133,9 +138,11 @@ impl VectorIndex for QdrantIndex { Ok(()) } - fn search(&mut self, collection: &str, vector: &[f32], limit: u32) -> Result> { + fn search(&mut self, collection: &str, vector: &[f32], limit: u32, within: Duration) -> Result> { + let http = self.http.bounded(within); let body = json!({ "vector": vector, "limit": limit, "with_payload": ["doc_id", "kind"], "with_vector": false }); - let (_, answer) = self.ask(Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?; + let (_, answer) = + self.ask_with(&http, Method::Post, &format!("/collections/{collection}/points/search"), Some(&body), &[200])?; let found = answer["result"].as_array().context("Qdrant's search has no result")?; found .iter() diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index c36699b..37d64f3 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -23,6 +23,9 @@ fn collection() -> String { collection_name(&slug(INSTANCE)) } +/// The bound the adapter tests give a call. +const TIME: Duration = Duration::from_secs(30); + /// A mind identity for tests that project entries without a mind behind them. const MIND: &str = "mind-commit-first"; @@ -308,25 +311,92 @@ fn change_label(index: &FakeIndex, change: impl FnOnce(&mut CollectionMeta)) { let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; change(meta); } - -/// The query embed is bounded by the search's deadline, one constant behind -/// both, so a query embed that cannot be recalled cannot hold the writes off -/// past the moment nobody is waiting for it. A bulk flush keeps its own bound. +/// Every call on the search path is bounded by what remains of the search's own +/// time: the identity reads, the label read, the embed and the query. A search +/// whose embed cannot be recalled therefore cannot hold the writes off past the +/// moment nobody is waiting for it. A bulk flush keeps its own, longer, bound. #[test] -fn the_query_embed_is_bounded_by_the_search_deadline_and_the_flush_by_its_own() { +fn every_call_on_the_search_path_is_bounded_by_the_searchs_time_and_the_flush_by_its_own() { let (_root, mind) = mind_with_documents(); let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); let mut projector = projected(&embedder, &index, &entries); - assert!(!embedder.state.lock().unwrap().bounds.is_empty()); - assert!(embedder.state.lock().unwrap().bounds.iter().all(|bound| *bound == BULK_EMBED_TIMEOUT), "the flushes"); - projector.search("anything", 3).unwrap(); - let bounds = embedder.state.lock().unwrap().bounds.clone(); - assert_eq!(bounds.last(), Some(&QUERY_EMBED_TIMEOUT), "the query"); + { + let state = embedder.state.lock().unwrap(); + assert!(!state.bounds.is_empty()); + let near = |bound: &Duration| *bound <= BULK_EMBED_TIMEOUT && *bound > BULK_EMBED_TIMEOUT - Duration::from_secs(60); + assert!(state.bounds.iter().all(near), "the flushes' embeds: {:?}", state.bounds); + assert!(state.identity_bounds.iter().all(near), "the reconciliation's and the flushes' identity reads: {:?}", state.identity_bounds); + } + embedder.state.lock().unwrap().bounds.clear(); + embedder.state.lock().unwrap().identity_bounds.clear(); + index.state.lock().unwrap().bounds.clear(); + + let within = Duration::from_secs(20); + projector.search("anything", 3, within).unwrap(); + let embedder = embedder.state.lock().unwrap(); + let index = index.state.lock().unwrap(); + assert_eq!(embedder.identity_bounds.len(), 2, "the model is read before and after the embed"); + assert_eq!(embedder.bounds.len(), 1); + assert_eq!(index.bounds.iter().map(|(call, _)| *call).collect::>(), ["describe", "search"]); + let bounds: Vec = embedder + .identity_bounds + .iter() + .chain(&embedder.bounds) + .copied() + .chain(index.bounds.iter().map(|(_, bound)| *bound)) + .collect(); + assert!(bounds.iter().all(|bound| !bound.is_zero() && *bound <= within), "{bounds:?}"); let served = crate::serve::ServeOptions::default().search_timeout; assert_eq!(served, SEARCH_DEADLINE); - assert!(QUERY_EMBED_TIMEOUT <= served); - assert!(QUERY_EMBED_TIMEOUT < BULK_EMBED_TIMEOUT); + assert!(SEARCH_DEADLINE < BULK_EMBED_TIMEOUT); +} + +/// A hung Ollama does not hold the worker past the search that asked: the +/// identity read never answers, and the search is answered, and the worker +/// free, when the search's own time has passed. A search that waited past its +/// time in the queue is answered without asking the embedder anything. A +/// ticket's time is `SEARCH_DEADLINE` from the moment it was asked. +#[test] +fn a_hung_embedder_holds_the_worker_no_longer_than_the_search_that_asked() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + + let asked = Instant::now(); + ask_sink(&mut sink, "the ticket's time", 3).unwrap(); + let due = lock(&sink.shared).searches.back().expect("the search is queued").due; + assert!(due >= asked + SEARCH_DEADLINE && due <= Instant::now() + SEARCH_DEADLINE, "due is SEARCH_DEADLINE after it was asked"); + answers(&mut sink, 1); + + embedder.state.lock().unwrap().hang_identity = true; + let started = Instant::now(); + let due = started + Duration::from_millis(400); + lock(&sink.shared).searches.push_back(Queued { ticket: SearchTicket(70), text: "hung".into(), top_k: 3, due }); + sink.shared.1.notify_all(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, SearchTicket(70)); + assert!(answered[0].1.is_err(), "{answered:?}"); + assert!(started.elapsed() < Duration::from_secs(5), "the worker was held for {:?}", started.elapsed()); + + embedder.state.lock().unwrap().hang_identity = false; + let reads = embedder.state.lock().unwrap().identity_bounds.len(); + lock(&sink.shared).searches.push_back(Queued { + ticket: SearchTicket(71), + text: "late".into(), + top_k: 3, + due: Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now), + }); + sink.shared.1.notify_all(); + let late = answers(&mut sink, 1); + let Err(why) = &late[0].1 else { panic!("{late:?}") }; + assert!(why.contains("has run out"), "{why}"); + assert_eq!(embedder.state.lock().unwrap().identity_bounds.len(), reads, "nothing was asked for a search nobody waits for"); + + let ticket = ask_sink(&mut sink, "the worker is free", 3).expect("the worker is still current"); + assert_eq!(answers(&mut sink, 1)[0].0, ticket); } /// The Ollama adapter enforces the bound it is given: an embed that is never @@ -543,7 +613,7 @@ const EMBED: &str = r#"{"embeddings":[[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1 fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { let ollama = stub(&[("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]); let mut embedder = OllamaEmbedder::new(&ollama.url, "m:1"); - assert_eq!(embedder.model_identity().unwrap(), ModelIdentity { name: "m:1".into(), digest: "ac6d".into(), dimensions: 4 }); + assert_eq!(embedder.model_identity(TIME).unwrap(), ModelIdentity { name: "m:1".into(), digest: "ac6d".into(), dimensions: 4 }); let vectors = embedder.embed(&["a".into(), "b".into(), "c".into(), "d".into()], Duration::from_secs(5)).unwrap(); assert_eq!(vectors.len(), 4); assert_eq!(vectors[0], vec![0.5, 1.0, 2.0, 3.0]); @@ -557,10 +627,10 @@ fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { ("GET /api/tags", 200, r#"{"models":[{"name":"m:1","digest":"ac6d"}]}"#), ("POST /api/show", 200, SHOW), ]); - assert_eq!(OllamaEmbedder::new(&bare.url, "m:1").model_identity().unwrap().digest, "ac6d"); + assert_eq!(OllamaEmbedder::new(&bare.url, "m:1").model_identity(TIME).unwrap().digest, "ac6d"); let mut absent = OllamaEmbedder::new(&ollama.url, "not-pulled:1"); - assert!(format!("{:#}", absent.model_identity().unwrap_err()).contains("does not list the model")); + assert!(format!("{:#}", absent.model_identity(TIME).unwrap_err()).contains("does not list the model")); let refusing = stub(&[("POST /api/embed", 500, r#"{"error":"boom"}"#)]); assert!(OllamaEmbedder::new(&refusing.url, "m:1").embed(&["a".into()], Duration::from_secs(5)).is_err()); } @@ -600,12 +670,12 @@ fn the_embedders_address_is_not_part_of_the_collections_compatibility() { fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { let name = collection(); let absent = stub(&[("GET /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); - assert_eq!(QdrantIndex::new(&absent.url).describe(&name).unwrap(), Described::Absent); + assert_eq!(QdrantIndex::new(&absent.url).describe(&name, TIME).unwrap(), Described::Absent); let unlabelled = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":null}}}"#)]); - assert_eq!(QdrantIndex::new(&unlabelled.url).describe(&name).unwrap(), Described::Unlabelled); + assert_eq!(QdrantIndex::new(&unlabelled.url).describe(&name, TIME).unwrap(), Described::Unlabelled); let voidbot = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":{"managedBy":"voidbot"}}}}"#)]); - assert_eq!(QdrantIndex::new(&voidbot.url).describe(&name).unwrap(), Described::Unlabelled); + assert_eq!(QdrantIndex::new(&voidbot.url).describe(&name, TIME).unwrap(), Described::Unlabelled); let labelled = stub(&[( "GET /collections/", @@ -613,7 +683,7 @@ fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","mind":"mind-commit-first","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, )]); assert_eq!( - QdrantIndex::new(&labelled.url).describe(&name).unwrap(), + QdrantIndex::new(&labelled.url).describe(&name, TIME).unwrap(), Described::Labelled(CollectionMeta { managed_by: "huginn".into(), instance: "yggdrasil".into(), @@ -943,14 +1013,14 @@ fn a_search_embeds_the_query_with_its_instruction_and_oversamples() { Hit { doc_id: "eureka-state:mystery:X".into(), kind: "mystery".into(), score: 0.8 }, ]; - let hits = projector.search("who owns the state", 3).unwrap(); + let hits = projector.search("who owns the state", 3, SEARCH_DEADLINE).unwrap(); assert_eq!(hits.len(), 1, "the unknown kind is left out"); assert_eq!(hits[0].0.id.0, "eureka-state:question:Q1"); assert_eq!(hits[0].0.kind, PipelineKind::Question); assert_eq!(hits[0].1, 0.9); let embedded = embedder.texts(); assert_eq!(embedded.last().unwrap(), &format!("Instruct: {QUERY_INSTRUCTION}\nQuery: who owns the state")); - projector.search("again", 100).unwrap(); + projector.search("again", 100, SEARCH_DEADLINE).unwrap(); let searches = index.state.lock().unwrap().searches.clone(); assert_eq!(searches.iter().map(|(name, length, limit)| (name.as_str(), *length, *limit)).collect::>(), [ (collection().as_str(), 4, 12), @@ -967,13 +1037,13 @@ fn a_search_refuses_when_the_index_is_not_reconciled_with_the_model() { let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - assert!(format!("{:#}", fresh.search("anything", 3).unwrap_err()).contains("being rebuilt")); + assert!(format!("{:#}", fresh.search("anything", 3, SEARCH_DEADLINE).unwrap_err()).contains("being rebuilt")); assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); let mut projector = projected(&embedder, &index, &entries); embedder.state.lock().unwrap().identity.digest = "d2".into(); let before = embedder.texts().len(); - assert!(format!("{:#}", projector.search("anything", 3).unwrap_err()).contains("model changed")); + assert!(format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()).contains("model changed")); assert_eq!(embedder.texts().len(), before, "the query was not embedded under the new model"); assert!(index.state.lock().unwrap().searches.is_empty()); assert!(projector.stale(), "and the reconciliation is to be redone"); @@ -991,7 +1061,7 @@ fn qdrant_search_sends_no_filter_and_reads_ids_kinds_and_scores() { 200, r#"{"result":[{"id":"p1","score":0.75,"payload":{"doc_id":"eureka-state:question:Q1","kind":"question"}},{"id":"p2","score":0.5,"payload":{"doc_id":"eureka-state:ruling:R1","kind":"ruling"}}]}"#, )]); - let hits = QdrantIndex::new(&answering.url).search(&name, &[0.5, 1.0, 2.0, 3.0], 12).unwrap(); + let hits = QdrantIndex::new(&answering.url).search(&name, &[0.5, 1.0, 2.0, 3.0], 12, TIME).unwrap(); assert_eq!( hits, [ @@ -1008,15 +1078,19 @@ fn qdrant_search_sends_no_filter_and_reads_ids_kinds_and_scores() { drop(seen); let nameless = stub(&[("POST /collections/", 200, r#"{"result":[{"id":"p","score":0.5,"payload":{"kind":"question"}}]}"#)]); - assert!(QdrantIndex::new(&nameless.url).search(&name, &[1.0], 1).is_err()); + assert!(QdrantIndex::new(&nameless.url).search(&name, &[1.0], 1, TIME).is_err()); let failing = stub(&[("POST /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); - assert!(QdrantIndex::new(&failing.url).search(&name, &[1.0], 1).is_err()); + assert!(QdrantIndex::new(&failing.url).search(&name, &[1.0], 1, TIME).is_err()); +} + +fn queued(ticket: u64, text: &str) -> Queued { + Queued { ticket: SearchTicket(ticket), text: text.into(), top_k: 3, due: Instant::now() + SEARCH_DEADLINE } } fn shared_with( inbox: Vec, mind: Option, - searches: Vec<(SearchTicket, String, u32)>, + searches: Vec, closed: bool, ) -> Lock { Arc::new(( @@ -1048,7 +1122,7 @@ fn an_idle_wait_ends_for_work_and_otherwise_at_its_deadline() { }; assert!(waits(&shared_with(vec![entry], None, Vec::new(), false)), "an entry"); assert!(waits(&shared_with(Vec::new(), Some(MIND.into()), Vec::new(), false)), "the mind's identity"); - assert!(waits(&shared_with(Vec::new(), None, vec![(SearchTicket(0), "q".into(), 3)], false)), "a search"); + assert!(waits(&shared_with(Vec::new(), None, vec![queued(0, "q")], false)), "a search"); assert!(waits(&shared_with(Vec::new(), None, Vec::new(), true)), "the sink closing"); let quiet = shared_with(Vec::new(), None, Vec::new(), false); @@ -1066,7 +1140,7 @@ fn an_idle_wait_ends_for_work_and_otherwise_at_its_deadline() { fn a_backoff_wait_is_the_whole_delay_whatever_arrives() { let (_root, mind) = mind_with_documents(); let entry = mind.index_entries(None).unwrap().remove(0); - let busy = shared_with(vec![entry], Some(MIND.into()), vec![(SearchTicket(0), "q".into(), 3)], false); + let busy = shared_with(vec![entry], Some(MIND.into()), vec![queued(0, "q")], false); let started = Instant::now(); wait_for_work(&busy, Wait::Backoff(Duration::from_millis(80))); assert!(started.elapsed() >= Duration::from_millis(80), "the delay was waited out although work was queued"); @@ -1080,8 +1154,7 @@ fn a_backoff_wait_is_the_whole_delay_whatever_arrives() { /// reason, when the worker fails: nothing is left to wait for its deadline. #[test] fn queued_searches_are_answered_with_why_when_the_worker_gives_up() { - let queued = vec![(SearchTicket(3), "a".into(), 1), (SearchTicket(4), "b".into(), 2)]; - let shared = shared_with(Vec::new(), None, queued, false); + let shared = shared_with(Vec::new(), None, vec![queued(3, "a"), queued(4, "b")], false); refuse_searches(&shared, "the index could not advance: down"); let guard = lock(&shared); assert!(guard.searches.is_empty()); @@ -1141,13 +1214,15 @@ fn queries(embedder: &FakeEmbedder) -> Vec { embedder.texts().into_iter().filter(|text| text.starts_with("Instruct: ")).collect() } -/// Only a current index answers. Being reconciled, being written to, failing -/// and being refused each refuse by name, and the count of documents pending -/// is in the words. +/// A search is refused only while the index is being rebuilt or cannot be +/// asked, and the words name the state and how many documents are pending. Lag +/// is not a rebuild: the sink and the projector ask the one predicate and both +/// let it through. #[test] -fn only_a_current_index_answers_a_search() { +fn a_search_is_refused_only_for_a_rebuild_or_a_failure() { use IndexStatus as S; assert_eq!(unsearchable(&S::Current), None); + assert_eq!(unsearchable(&S::Behind { pending: 5 }), None, "lag is waited out, not refused"); let says = |status: S, wanted: &[&str]| { let why = unsearchable(&status).unwrap_or_else(|| panic!("{status:?} answered")); for word in wanted { @@ -1155,65 +1230,68 @@ fn only_a_current_index_answers_a_search() { } }; says(S::Reconciling { pending: 7 }, &["being rebuilt", "reconciling", "pending: 7"]); - says(S::Behind { pending: 5 }, &["being rebuilt", "behind", "pending: 5"]); says(S::Failing { pending: 3, attempts: 2, error: "boom".into() }, &["failing", "boom", "pending: 3"]); says(S::Refused { pending: 4, attempts: 2, reason: "not ours".into() }, &["refused", "not ours", "pending: 4"]); - // The projector never answers from a half-filled collection, and never - // touches the embedder or the vector store to find out. + // The sink: what the worker last published, and the documents admitted + // since, which are lag whatever the published state says of a current index. + let sink_in = |status: S, inbox: usize| { + let shared = shared_with(entries_of(inbox), None, Vec::new(), false); + lock(&shared).status = status; + WorkerSink { shared, identified: true } + }; + assert!(ask_sink(&mut sink_in(S::Current, 2), "anything", 3).is_ok(), "admitted a moment ago and not yet absorbed"); + assert!(ask_sink(&mut sink_in(S::Behind { pending: 4 }, 0), "anything", 3).is_ok(), "the worker is writing"); + let refusal = ask_sink(&mut sink_in(S::Reconciling { pending: 4 }, 2), "anything", 3).unwrap_err(); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 6"), "{refusal}"); + let refusal = ask_sink(&mut sink_in(S::Failing { pending: 1, attempts: 1, error: "boom".into() }, 0), "anything", 3).unwrap_err(); + assert!(refusal.contains("failing") && refusal.contains("boom"), "{refusal}"); + + // The projector: a rebuild refuses without touching the ports to find out. let (_root, mind) = mind_with_documents(); let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - projector.want(entries[3..].iter().cloned()); - let embedded = embedder.texts().len(); - let refusal = format!("{:#}", projector.search("anything", 3).unwrap_err()); - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 1"), "{refusal}"); - assert_eq!(embedder.texts().len(), embedded); - assert!(index.state.lock().unwrap().searches.is_empty()); - drain(&mut projector); - assert!(projector.search("anything", 3).is_ok(), "once it has caught up it answers"); - - // The sink refuses before the worker is asked, while the worker is inside a batch. - let gate = Gate::shut(); - let mut sink = WorkerSink::spawn( - FakeEmbedder::new("d1").behind(&gate), - FakeIndex::default(), - &slug(INSTANCE), - Some(MIND.into()), - entries.clone(), - quick(), - ); - gate.wait_arrived(1); - let refusal = ask_sink(&mut sink, "anything", 3).unwrap_err(); + let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + fresh.want(entries.iter().cloned()); + let refusal = format!("{:#}", fresh.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); - gate.open(); - sink.wait_status(|status| *status == IndexStatus::Current); - assert!(ask_sink(&mut sink, "anything", 3).is_ok()); + assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); + + // Lag it lets through: catching up is the worker's, before the search is taken. + let mut lagging = projected(&embedder, &index, &entries[..3]); + lagging.want(entries[3..].iter().cloned()); + assert_eq!(lagging.progress_status(), IndexStatus::Behind { pending: 1 }); + assert!(lagging.search("anything", 3, SEARCH_DEADLINE).is_ok()); } -/// A search that fails after it was accepted has found the index not what it -/// claimed: the reconciliation is forgotten, so the status stops reading -/// `Current` and the worker verifies the model and the collection again. +/// A search that fails for its own reasons (the vector store errs, the +/// embedder is down, an identity read times out) fails that search alone: the +/// reconciliation stands, the status stays `Current`, the collection is not +/// listed again, and the next search is asked and answered. #[test] -fn a_search_that_fails_forgets_the_reconciliation() { +fn a_failed_search_fails_that_search_alone() { let (_root, mind) = mind_with_documents(); let entries = mind.index_entries(None).unwrap(); - let breaks: [(&str, Box); 2] = [ - ("the vector store fails the search", Box::new(|_, index| index.state.lock().unwrap().fail_search = true)), - ("the embedder is down", Box::new(|embedder, _| embedder.state.lock().unwrap().down = true)), + type Fault = Box; + let faults: [(&str, Fault); 3] = [ + ("the vector store fails the search", Box::new(|_, index, on| index.state.lock().unwrap().fail_search = on)), + ("the embedder is down", Box::new(|embedder, _, on| embedder.state.lock().unwrap().down = on)), + ("the identity read never answers", Box::new(|embedder, _, on| embedder.state.lock().unwrap().hang_identity = on)), ]; - for (what, breaks) in breaks { + for (what, fault) in faults { let (embedder, index) = pair(); let mut projector = projected(&embedder, &index, &entries); - assert_eq!(projector.progress_status(), IndexStatus::Current); - breaks(&embedder, &index); - assert!(projector.search("anything", 3).is_err(), "{what}"); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 0 }, "{what}"); - assert!(projector.stale(), "{what}"); + let listed = index.state.lock().unwrap().listed; + fault(&embedder, &index, true); + assert!(projector.search("anything", 3, Duration::from_millis(50)).is_err(), "{what}"); + assert_eq!(projector.progress_status(), IndexStatus::Current, "{what}"); + assert!(!projector.stale(), "{what}"); + fault(&embedder, &index, false); + assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_ok(), "{what}: the next search is answered"); + assert_eq!(index.state.lock().unwrap().listed, listed, "{what}: the collection was not listed again"); } - // In the worker the next step is the verification, with nothing to prompt it. + // In the worker the failed search is answered and nothing else moves. let (embedder, index) = pair(); let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, quick()); sink.wait_status(|status| *status == IndexStatus::Current); @@ -1223,7 +1301,24 @@ fn a_search_that_fails_forgets_the_reconciliation() { let failed = answers(&mut sink, 1); assert_eq!(failed[0].0, ticket); assert!(failed[0].1.is_err()); - eventually("the worker listing the collection again", || index.state.lock().unwrap().listed > listed); + index.state.lock().unwrap().fail_search = false; + let next = ask_sink(&mut sink, "anything", 3).expect("the index is still current, not being rebuilt"); + let answered = answers(&mut sink, 1); + assert_eq!((answered[0].0, answered[0].1.is_ok()), (next, true)); + assert_eq!(index.state.lock().unwrap().listed, listed, "the worker did not list the collection again"); +} + +/// A model the search finds changed is a rebuild, and its refusal names the +/// state and how many documents are pending, as every other refusal does. +#[test] +fn a_model_changed_refusal_names_the_state_and_the_pending_count() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + let refusal = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); + assert!(refusal.contains("model changed") && refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); } /// A search asks the store what the collection says about itself: a label for @@ -1252,7 +1347,7 @@ fn a_search_re_reads_the_collections_label() { let mut projector = projected(&embedder, &index, &entries); change(index.state.lock().unwrap().collections.get_mut(&collection()).unwrap()); let embedded = embedder.texts().len(); - let refusal = format!("{:#}", projector.search("anything", 3).unwrap_err()); + let refusal = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); assert!(refusal.contains("no longer the one this mind's index was built as"), "{what}: {refusal}"); assert_eq!(embedder.texts().len(), embedded, "{what}: nothing was embedded"); assert!(index.state.lock().unwrap().searches.is_empty(), "{what}"); @@ -1261,7 +1356,7 @@ fn a_search_re_reads_the_collections_label() { let (embedder, index) = pair(); let mut projector = projected(&embedder, &index, &entries); index.state.lock().unwrap().collections.clear(); - assert!(projector.search("anything", 3).is_err(), "no collection"); + assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_err(), "no collection"); assert!(projector.stale()); } @@ -1278,7 +1373,7 @@ fn a_model_that_changes_while_the_query_is_embedded_is_refused() { let mut projector = projected(&embedder, &index, &entries); gate.shut_again(); let searching = std::thread::spawn(move || { - let found = projector.search("anything", 3); + let found = projector.search("anything", 3, SEARCH_DEADLINE); (projector, found) }); gate.wait_arrived(1); @@ -1372,33 +1467,72 @@ fn one_queued_search_is_served_per_turn() { let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); let mut projector = projected(&embedder, &index, &entries); - let queued = vec![(SearchTicket(1), "a".into(), 3), (SearchTicket(2), "b".into(), 3), (SearchTicket(3), "c".into(), 3)]; - let shared = shared_with(Vec::new(), None, queued, false); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a"), queued(2, "b"), queued(3, "c")], false); serve_search(&mut projector, &shared); let guard = lock(&shared); - assert_eq!(guard.searches.iter().map(|(ticket, ..)| *ticket).collect::>(), [SearchTicket(2), SearchTicket(3)]); + assert_eq!(guard.searches.iter().map(|queued| queued.ticket).collect::>(), [SearchTicket(2), SearchTicket(3)]); assert_eq!(guard.found.iter().map(|(ticket, _)| *ticket).collect::>(), [SearchTicket(1)]); } -/// Searches are answered between batches, not after the last: a search waiting -/// while the first of two batches is written is answered, refused for the -/// documents still pending, before the second batch is embedded. +/// A search reads its own writes. Asked while a batch is in the embedder and +/// eight more documents wait, it is accepted, waits behind them, and is +/// answered only once every one is in the collection. #[test] -fn a_waiting_search_is_answered_between_batches() { +fn a_search_waits_for_the_writes_ahead_of_it_and_reads_them() { let gate = Gate::shut(); let (_, index) = pair(); let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), quick()); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), quick()); gate.wait_arrived(1); - lock(&sink.shared).searches.push_back((SearchTicket(9), "anything".into(), 3)); + let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); gate.open(); let answered = answers(&mut sink, 1); - assert_eq!(answered[0].0, SearchTicket(9)); - let Err(refusal) = &answered[0].1 else { panic!("a search was answered from a half-filled collection: {answered:?}") }; - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 8"), "{refusal}"); + assert_eq!(answered[0].0, ticket); + assert!(answered[0].1.is_ok(), "{answered:?}"); + assert_eq!(index.state.lock().unwrap().held_when_searched, [BATCH + 8], "the search saw every document admitted before it"); sink.wait_status(|status| *status == IndexStatus::Current); } +/// The worker takes admissions before the search, and leaves the search queued +/// while the writes ahead of it are unfinished: a search asked with a document +/// still in the inbox absorbs it and waits, and once the index has caught up +/// it is answered. An index being rebuilt is not waited on: the search is taken +/// and refused, naming the state and the pending count. +#[test] +fn the_worker_absorbs_admissions_before_a_search_and_waits_for_lag_but_not_for_a_rebuild() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + let shared = shared_with(vec![entries[3].clone()], None, vec![queued(1, "a")], false); + + serve_search(&mut projector, &shared); + { + let guard = lock(&shared); + assert!(guard.inbox.is_empty(), "the admission ahead of the search was absorbed"); + assert_eq!(guard.searches.len(), 1, "the search waits for it to be written"); + assert!(guard.found.is_empty()); + } + assert_eq!(projector.pending(), 1); + + drain(&mut projector); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(guard.searches.is_empty()); + assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); + assert_eq!(index.state.lock().unwrap().held_when_searched, [4]); + drop(guard); + + // A rebuild: the projector has forgotten its reconciliation with the entries owed. + let mut rebuilding = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + rebuilding.want(entries.iter().cloned()); + let shared = shared_with(Vec::new(), None, vec![queued(2, "b")], false); + serve_search(&mut rebuilding, &shared); + let guard = lock(&shared); + let [(SearchTicket(2), Err(refusal))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); +} + /// A search that was queued when the worker failed is answered with why, by /// the worker's own failure path. #[test] @@ -1408,13 +1542,13 @@ fn a_search_queued_before_the_worker_fails_is_answered_with_why() { let embedder = FakeEmbedder::new("d1").behind(&gate); let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries_of(1), quick()); gate.wait_arrived(1); - lock(&sink.shared).searches.push_back((SearchTicket(5), "anything".into(), 3)); + lock(&sink.shared).searches.push_back(queued(5, "anything")); embedder.state.lock().unwrap().down = true; gate.open(); let answered = answers(&mut sink, 1); assert_eq!(answered[0].0, SearchTicket(5)); let Err(why) = &answered[0].1 else { panic!("{answered:?}") }; - assert!(why.contains("the index could not advance") && why.contains("the embedder is down"), "{why}"); + assert!(why.contains("the semantic index is failing") && why.contains("pending: 1") && why.contains("the embedder is down"), "{why}"); } /// While a failure is being reported, the report's pending count is the one diff --git a/crates/huginn-daemon/src/semantic_tests.rs b/crates/huginn-daemon/src/semantic_tests.rs index 6880009..9a5646b 100644 --- a/crates/huginn-daemon/src/semantic_tests.rs +++ b/crates/huginn-daemon/src/semantic_tests.rs @@ -461,12 +461,13 @@ fn a_search_that_timed_out_while_queued_is_never_embedded() { assert!(embedded[0].ends_with("Query: first") && embedded[1].ends_with("Query: third"), "{embedded:?}"); } -/// While the index is rebuilding, a semantic query is refused with an -/// `Unavailable` that names the state and how many documents are pending, and -/// the embedder is not asked for it. When the index has caught up the same -/// query is answered. +/// A search asked while the index lags the mind is not refused: it waits behind +/// the writes ahead of it, through the real serve loop, and reads them. The +/// worker is inside its first batch with the rest of the mind still to write; +/// the reply does not come until the gate opens, and it comes with every +/// document already in the collection. #[test] -fn a_search_while_the_index_is_being_rebuilt_is_refused_and_asks_the_embedder_nothing() { +fn a_search_asked_while_the_index_lags_waits_for_the_writes_ahead_of_it_and_reads_them() { let gate = Gate::shut(); let embedder = FakeEmbedder::new("d1").behind(&gate); let harness = Harness::start(shelf(), embedder, FakeIndex::default(), Duration::from_secs(30)); @@ -474,14 +475,17 @@ fn a_search_while_the_index_is_being_rebuilt_is_refused_and_asks_the_embedder_no let mut client = harness.client(); let IndexStatus::Behind { pending } = index_status(&mut client, "m-status") else { panic!("the worker is inside its first batch") }; assert!(pending > 0); + harness.index.state.lock().unwrap().hits = vec![hit("ruling", id("ruling", "R4"), 0.5)]; - let refused = ask(&mut client, "m-behind", &semantic(standing(), 5)); - let HuginnMindResponse::Refused(MindRefusal::Unavailable { detail }) = &refused else { panic!("{refused:?}") }; - assert!(detail.contains("being rebuilt") && detail.contains(&format!("pending: {pending}")), "{detail}"); - assert!(harness.embedder.texts().is_empty(), "nothing was embedded, not even the documents, for a refused search"); + send(&mut client, "m-behind", &semantic(standing(), 5)); + assert!(receive(&mut client, "m-behind", 100).is_none(), "the search waits: it is neither refused nor answered from a half-filled collection"); gate.open(); + let answered = receive(&mut client, "m-behind", 3000).expect("the search is answered once the index has caught up"); + assert_eq!(ids(&answered), vec![id("ruling", "R4")]); until_index(&mut client, "Current", |status| *status == IndexStatus::Current); - harness.index.state.lock().unwrap().hits = vec![hit("ruling", id("ruling", "R4"), 0.5)]; - assert_eq!(ids(&ask(&mut client, "m-current", &semantic(standing(), 5))), vec![id("ruling", "R4")]); + let state = harness.index.state.lock().unwrap(); + let written = state.collections[&collection_name(&slug(INSTANCE))].points.len(); + assert_eq!(state.held_when_searched, [written], "the search saw every document"); + assert!(written >= pending as usize); } diff --git a/crates/huginn-daemon/tests/live_index_smoke.rs b/crates/huginn-daemon/tests/live_index_smoke.rs index 7f6b4d4..85fabc2 100644 --- a/crates/huginn-daemon/tests/live_index_smoke.rs +++ b/crates/huginn-daemon/tests/live_index_smoke.rs @@ -44,7 +44,7 @@ fn the_adapters_embed_write_query_and_clean_up_a_scratch_collection() { let dotted = format!("{plain}.dotted"); let mut embedder = OllamaEmbedder::new(&ollama, &model); - let identity = embedder.model_identity().expect("the model is listed"); + let identity = embedder.model_identity(Duration::from_secs(60)).expect("the model is listed"); println!("identity: {identity:?}"); let texts = ["The organ owns the mind.".to_string(), "A recipe for lentil soup.".to_string()]; let vectors = embedder.embed(&texts, Duration::from_secs(60)).expect("two texts embed"); @@ -63,9 +63,9 @@ fn the_adapters_embed_write_query_and_clean_up_a_scratch_collection() { let mut store = QdrantIndex::new(&qdrant); let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { for name in [&plain, &dotted] { - assert_eq!(store.describe(name).unwrap(), Described::Absent); + assert_eq!(store.describe(name, Duration::from_secs(60)).unwrap(), Described::Absent); store.recreate(name, &meta).unwrap(); - assert_eq!(store.describe(name).unwrap(), Described::Labelled(meta.clone())); + assert_eq!(store.describe(name, Duration::from_secs(60)).unwrap(), Described::Labelled(meta.clone())); assert!(store.ids(name).unwrap().is_empty()); let points: Vec = texts .iter() @@ -97,7 +97,7 @@ fn the_adapters_embed_write_query_and_clean_up_a_scratch_collection() { let asked = format!("Instruct: {QUERY_INSTRUCTION} Query: Who is responsible for keeping the memory?"); let query = embedder.embed(&[asked], Duration::from_secs(60)).expect("the query embeds").remove(0); - let hits = store.search(name, &query, 2).unwrap(); + let hits = store.search(name, &query, 2, Duration::from_secs(60)).unwrap(); assert_eq!(hits.len(), 2, "{hits:?}"); assert_eq!((hits[0].doc_id.as_str(), hits[0].kind.as_str()), ("smoke:doc:0", "question"), "{hits:?}"); assert!(hits[0].score > hits[1].score, "{hits:?}"); From b48f0cb5ece85758d708634da87b26cf1f91eb85 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:43:46 +0200 Subject: [PATCH 020/520] Give the model-changed refusal test documents still owed The refusal names the pending count; with every document written the count is zero and the test proved nothing about it. Co-Authored-By: Claude Opus 5.5 --- crates/huginn-daemon/src/index/tests.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index 37d64f3..76c3ddd 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -1315,10 +1315,11 @@ fn a_model_changed_refusal_names_the_state_and_the_pending_count() { let (_root, mind) = mind_with_documents(); let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); + let mut projector = projected(&embedder, &index, &entries[..3]); + projector.want(entries[3..].iter().cloned()); embedder.state.lock().unwrap().identity.digest = "d2".into(); let refusal = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); - assert!(refusal.contains("model changed") && refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); + assert!(refusal.contains("model changed") && refusal.contains("being rebuilt") && refusal.contains("pending: 1"), "{refusal}"); } /// A search asks the store what the collection says about itself: a label for From e99dc4404e13c5c493f654c2efd9bab0a8ffb119 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:28:22 +0200 Subject: [PATCH 021/520] Decide and take a search under one guard; a refill is a rebuild; a first build is lag S1. serve_search absorbed the inbox and judged pending under one lock and popped the search under a second, so an admission acknowledged before the search was asked could be missed. The absorb, the judgement and the take are now one critical section, and take_search never takes a search while the inbox is non-empty or anything ahead of it is owed. S2. The projector carries Build (Unbuilt, Built, Rebuilding). A collection made again (label mismatch, or lost after it was once built) is Rebuilding until the entries it owes are written; settle() is the one place that clears it, so a failure mid-refill does not end the rebuild. Rebuilding reports Reconciling. unsearchable stays the only refusal predicate. S3. A mind whose index was never built is Unbuilt: its first build reports Behind, so the search waits and answers whichever side looked first. Only a rebuild of an index that exists refuses. S5. ServeOptions::search_timeout is deleted. The sink owns the deadline (Backoff::search_deadline, stamped on every search) and the serve loop asks the daemon for it (IndexSink::search_deadline), so the loop's wait and the worker's bound are one value. A mind with no identity and documents owed is Reconciling, so a search is taken and refused rather than left for a spinning worker. S4. A test pins that a later call's bound shrinks as the search spends time. Co-Authored-By: Claude Opus 5.5 --- crates/huginn-daemon/src/daemon.rs | 20 ++ crates/huginn-daemon/src/index/fakes.rs | 9 +- crates/huginn-daemon/src/index/mod.rs | 146 ++++++++--- crates/huginn-daemon/src/index/tests.rs | 292 ++++++++++++++++++++- crates/huginn-daemon/src/semantic_tests.rs | 6 +- crates/huginn-daemon/src/serve.rs | 16 +- 6 files changed, 429 insertions(+), 60 deletions(-) diff --git a/crates/huginn-daemon/src/daemon.rs b/crates/huginn-daemon/src/daemon.rs index 8b831a3..ffbf813 100644 --- a/crates/huginn-daemon/src/daemon.rs +++ b/crates/huginn-daemon/src/daemon.rs @@ -25,6 +25,9 @@ pub trait IndexSink { /// Starts a search for the documents nearest `text`, at most `top_k` of /// them wanted. `Err` says why the index cannot be asked now. fn search(&mut self, text: &str, top_k: u32) -> Result; + /// How long after it was asked a search is worth answering: the index's + /// own bound on it, which the loop waiting for the answer must share. + fn search_deadline(&self) -> std::time::Duration; /// Every search that finished since the last call, by ticket: its /// candidates and scores, or why it failed. fn searched(&mut self) -> Vec<(SearchTicket, Result)>; @@ -174,6 +177,11 @@ impl> Daemon { } } + /// How long a search is worth waiting for. + pub fn search_deadline(&self) -> std::time::Duration { + self.index.search_deadline() + } + /// Searches the index has finished since this was last called. pub fn searched(&mut self) -> Vec<(SearchTicket, Result)> { self.index.searched() @@ -466,6 +474,10 @@ pub(crate) mod tests { Err("this index cannot search".into()) } + fn search_deadline(&self) -> std::time::Duration { + std::time::Duration::ZERO + } + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { Vec::new() } @@ -502,6 +514,10 @@ pub(crate) mod tests { Err("this index cannot search".into()) } + fn search_deadline(&self) -> std::time::Duration { + std::time::Duration::ZERO + } + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { Vec::new() } @@ -524,6 +540,10 @@ pub(crate) mod tests { Err("this index cannot search".into()) } + fn search_deadline(&self) -> std::time::Duration { + std::time::Duration::ZERO + } + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { Vec::new() } diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs index 61c98a2..20bcc8c 100644 --- a/crates/huginn-daemon/src/index/fakes.rs +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -71,6 +71,8 @@ pub(crate) struct EmbedderState { /// Whether an identity read is never answered: it fails when its bound /// has passed, as an adapter that honours its bound does. pub(crate) hang_identity: bool, + /// How long an identity read takes to answer, when it answers. + pub(crate) identity_delay: Duration, } #[derive(Clone)] @@ -81,7 +83,7 @@ pub(crate) struct FakeEmbedder { impl FakeEmbedder { pub(crate) fn new(digest: &str) -> Self { - let state = EmbedderState { identity: identity(digest), down: false, embedded: Vec::new(), bounds: Vec::new(), vector_length: None, identity_bounds: Vec::new(), hang_identity: false }; + let state = EmbedderState { identity: identity(digest), down: false, embedded: Vec::new(), bounds: Vec::new(), vector_length: None, identity_bounds: Vec::new(), hang_identity: false, identity_delay: Duration::ZERO }; Self { state: Arc::new(Mutex::new(state)), gate: None } } @@ -107,7 +109,10 @@ impl Embedder for FakeEmbedder { if state.down { bail!("the embedder is down"); } - Ok(state.identity.clone()) + let (identity, delay) = (state.identity.clone(), state.identity_delay); + drop(state); + std::thread::sleep(delay); + Ok(identity) } fn embed(&mut self, texts: &[String], within: Duration) -> Result>> { diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs index 367e675..30a7b4c 100644 --- a/crates/huginn-daemon/src/index/mod.rs +++ b/crates/huginn-daemon/src/index/mod.rs @@ -62,23 +62,27 @@ //! - A search runs on the worker between batches, so it waits for the batch in //! flight, and at most one runs between two steps of the writes. The serve //! loop never waits: it holds the reply and answers when the worker has -//! (`ServeOptions::search_timeout` bounds the wait), and takes back a search +//! (`IndexSink::search_deadline` bounds the wait), and takes back a search //! it has stopped waiting for. At most `SEARCHES_QUEUED_MAX` wait; one more is //! refused. A search whose embed is already running cannot be recalled, so it //! can hold the writes off for as long as it has left: the ticket carries the -//! moment `SEARCH_DEADLINE` after it was asked, and every call the search +//! moment `Backoff::search_deadline` after it was asked, and every call the search //! makes (the model's identity, the collection's label, the embed, the //! query) is bounded by what remains of it. //! - A search reads its own writes. Admissions and searches meet under one -//! lock, admissions first: a search waits in the queue while the worker -//! absorbs and writes what was admitted before it, and is answered once the -//! index has caught up. Lag is not a rebuild and refuses nothing. Under a -//! steady stream of admissions a search may never see the index caught up; -//! its deadline then answers it. +//! lock, and the worker judges a search and takes it in one critical +//! section, admissions first: a search waits in the queue while anything +//! ahead of it is owed (documents in the inbox, documents to write, a first +//! reconciliation to make), and is answered once the index has caught up. Lag +//! is not a rebuild and refuses nothing, and a mind's first build is lag: +//! there is no earlier index to protect. Under a steady stream of admissions +//! a search may never see the index caught up; its deadline then answers it. //! - A search is refused only while the index is being rebuilt or cannot be -//! asked: `Reconciling` (a re-pulled model, a lost collection, a mind's first -//! admission), `Failing` or `Refused`. The refusal is an `Unavailable` that -//! names the state and how many documents are pending; it is never answered +//! asked: `Reconciling` (a collection made again because the model, its +//! digest or the text version changed or because it was lost, until every +//! document it owes is written again), `Failing` or `Refused`. The refusal +//! is an `Unavailable` that names the state and how many documents are +//! pending; it is never answered //! from a collection being rebuilt. A search that fails for its own reasons //! (a timeout, a store error) fails that search alone and the //! reconciliation stands. Only what the check-then-embed finds changed (the @@ -109,10 +113,10 @@ pub const MANAGED_BY: &str = "huginn"; /// Documents embedded and written per call. pub const BATCH: usize = 32; -/// How long a search may take, queue wait included: the serve loop's default -/// `ServeOptions::search_timeout`. Every bound on the search path derives from -/// the moment this long after the search was asked, so the two cannot drift -/// apart. +/// How long a search may take, queue wait included, by default +/// (`Backoff::search_deadline`). The sink stamps it on every search when it is +/// asked, and the serve loop asks the sink for it (`IndexSink::search_deadline`), +/// so the loop's wait and the worker's bound are one value. pub const SEARCH_DEADLINE: Duration = Duration::from_secs(30); /// The bound on a bulk index-flush embed, the identity and label reads around @@ -277,6 +281,22 @@ impl Budget { } } +/// Whether the collection holds an index worth protecting, and whether it is +/// being made again. The state that marks a rebuild lasts until the entries it +/// owes have been written; nothing else clears it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Build { + /// No reconciliation has yet found or made this mind's collection. Its + /// first build has nothing stale to protect, so it is lag. + Unbuilt, + /// The collection was built by this model and only lags the mind. + Built, + /// The collection was made again (another model, digest or text version, + /// or lost while the worker ran) and is refilled from what the projector + /// knows: until every entry it owes is written, it answers nothing. + Rebuilding, +} + /// The projection's state: the entries not yet known to be in the collection, /// and, once reconciled, the length of vector it accepts. pub struct Projector { @@ -294,6 +314,7 @@ pub struct Projector { /// The model the collection was reconciled under; `None` until it is, and /// again whenever that reconciliation is forgotten. reconciled: Option, + build: Build, } impl Projector { @@ -307,6 +328,7 @@ impl Projector { known: BTreeMap::new(), wanted: BTreeSet::new(), reconciled: None, + build: Build::Unbuilt, } } @@ -398,7 +420,12 @@ impl Projector { self.embedder.model_identity(BULK_EMBED_TIMEOUT).context("reading the embedding model's identity")?; let wanted = self.collection_meta(&mind, &identity); match self.index.describe(&self.collection, INDEX_CALL_TIMEOUT).context("describing the collection")? { - Described::Absent => self.recreate(&wanted)?, + Described::Absent => { + // A collection lost while the worker ran was an index; one that + // was never made is a first build. + let build = if self.build == Build::Unbuilt { Build::Built } else { Build::Rebuilding }; + self.recreate(&wanted, build)?; + } Described::Unlabelled => { return Err(StepError::Refused(format!( "collection {} exists and carries no Huginn metadata; it is not this organ's to rewrite", @@ -413,12 +440,14 @@ impl Projector { self.collection, found.managed_by, found.instance, found.mind, self.instance, mind ))); } - Described::Labelled(found) if found != wanted => self.recreate(&wanted)?, + Described::Labelled(found) if found != wanted => self.recreate(&wanted, Build::Rebuilding)?, + Described::Labelled(_) if self.build == Build::Unbuilt => self.build = Build::Built, Described::Labelled(_) => {} } let present = self.index.ids(&self.collection).context("listing the collection's points")?; self.wanted = self.known.keys().filter(|document_id| !present.contains(&point_id(document_id))).cloned().collect(); self.reconciled = Some(identity); + self.settle(); Ok(()) } @@ -436,8 +465,20 @@ impl Projector { } } - fn recreate(&mut self, meta: &CollectionMeta) -> Result<()> { - self.index.recreate(&self.collection, meta).context("creating the collection") + /// Makes the collection again. `build` is what it is once made, recorded + /// with the collection so a failure before the refill leaves it recorded. + fn recreate(&mut self, meta: &CollectionMeta, build: Build) -> Result<()> { + self.index.recreate(&self.collection, meta).context("creating the collection")?; + self.build = build; + Ok(()) + } + + /// A rebuild ends when the entries it owes are all written: the one place + /// it is cleared. + fn settle(&mut self) { + if self.build == Build::Rebuilding && self.wanted.is_empty() { + self.build = Build::Built; + } } /// The one embed-and-upsert routine, for startup and live entries alike: @@ -483,6 +524,7 @@ impl Projector { for entry in &batch { self.wanted.remove(&entry.id.id.0); } + self.settle(); Ok(()) } @@ -537,7 +579,7 @@ impl Projector { } let budget = Budget::new(within); let Some(reconciled) = self.reconciled.clone() else { - bail!("the mind has no identity yet, so it has no index"); + bail!("the index has not been built yet"); }; let dimensions = reconciled.dimensions as usize; let query = format!("Instruct: {QUERY_INSTRUCTION}\nQuery: {text}"); @@ -566,8 +608,12 @@ impl Projector { pub fn progress_status(&self) -> IndexStatus { match (&self.reconciled, self.pending()) { (None, 0) if self.mind.is_none() => IndexStatus::Current, + // The first build of an identified mind has nothing stale to + // protect: the wait for it is lag. + (None, pending) if self.mind.is_some() && self.build == Build::Unbuilt => IndexStatus::Behind { pending }, (None, pending) => IndexStatus::Reconciling { pending }, (Some(_), 0) => IndexStatus::Current, + (Some(_), pending) if self.build == Build::Rebuilding => IndexStatus::Reconciling { pending }, (Some(_), pending) => IndexStatus::Behind { pending }, } } @@ -585,17 +631,19 @@ impl Projector { /// The worker's clock: the retry delay starts at `initial`, doubles per /// consecutive failure and stops at `max`; an idle worker verifies the model -/// and the collection again every `recheck`. +/// and the collection again every `recheck`; a search is worth waiting for +/// `search_deadline` after it was asked. #[derive(Clone, Copy, Debug)] pub struct Backoff { pub initial: Duration, pub max: Duration, pub recheck: Duration, + pub search_deadline: Duration, } impl Default for Backoff { fn default() -> Self { - Self { initial: Duration::from_secs(30), max: Duration::from_secs(600), recheck: Duration::from_secs(60) } + Self { initial: Duration::from_secs(30), max: Duration::from_secs(600), recheck: Duration::from_secs(60), search_deadline: SEARCH_DEADLINE } } } @@ -719,20 +767,41 @@ fn unsearchable(status: &IndexStatus) -> Option { /// the worker returns to its writes after each, so a queue of searches cannot /// hold the index's writes off for the sum of their embeddings. /// -/// Admissions come first: the inbox is absorbed before the search is taken, so -/// every document committed before the search was asked is wanted, and a search -/// behind lag stays queued until the worker's writes have caught up. An index -/// being rebuilt does not wait: the search is taken and refused. The search's -/// calls are bounded by what remains of the time it was given when it was asked. +/// Admissions come first, and the judgement and the take are one critical +/// section: the inbox is absorbed and the search taken under one guard, so a +/// document acknowledged before the search was asked is wanted when the search +/// is judged, and none can slip in between. A search is taken only with nothing +/// ahead of it (`take_search`). The search's calls are bounded by what remains +/// of the time it was given when it was asked. fn serve_search(projector: &mut Projector, shared: &Lock) { - if !absorb(projector, shared, None) || (projector.pending() > 0 && !projector.stale()) { + let mut guard = lock(shared); + if !absorb_into(projector, &mut guard, None) { return; } - let Some(Queued { ticket, text, top_k, due }) = lock(shared).searches.pop_front() else { return }; + #[cfg(test)] + tests::after_absorb(&mut guard); + let Some(Queued { ticket, text, top_k, due }) = take_search(projector, &mut guard) else { return }; + drop(guard); let found = projector.search(&text, top_k, due.saturating_duration_since(Instant::now())).map_err(|error| format!("{error:#}")); lock(shared).found.push((ticket, found)); } +/// The search at the front of the queue, if it may be served now. Never while +/// a document sits in the inbox, and never while anything ahead of it is +/// owed (documents to write, a reconciliation to make) unless the index +/// cannot be asked at all, in which case the search is taken to be refused: +/// waiting would not change the answer. +fn take_search(projector: &Projector, guard: &mut Shared) -> Option { + if !guard.inbox.is_empty() { + return None; + } + let caught_up = projector.pending() == 0 && !projector.stale(); + if !caught_up && unsearchable(&projector.progress_status()).is_none() { + return None; + } + guard.searches.pop_front() +} + /// Answers every queued search with why the index cannot: they were accepted /// while it was healthy and it has failed since. fn refuse_searches(shared: &Lock, why: &str) { @@ -756,7 +825,16 @@ fn absorb( shared: &Lock, failing: Option<(&StepError, u32)>, ) -> bool { - let mut guard = lock(shared); + absorb_into(projector, &mut lock(shared), failing) +} + +/// `absorb` under a guard the caller already holds, for the caller that must +/// decide something in the same critical section. +fn absorb_into( + projector: &mut Projector, + guard: &mut Shared, + failing: Option<(&StepError, u32)>, +) -> bool { if guard.closed { return false; } @@ -830,6 +908,8 @@ pub struct WorkerSink { shared: Lock, /// Whether the worker has been told the mind's identity. identified: bool, + /// How long after it was asked a search is worth answering. + deadline: Duration, } impl WorkerSink { @@ -866,7 +946,7 @@ impl WorkerSink { )); let worker = Arc::clone(&shared); std::thread::spawn(move || work(projector, worker, backoff)); - Self { shared, identified } + Self { shared, identified, deadline: backoff.search_deadline } } /// Blocks until the health satisfies `predicate`, and returns it. For @@ -944,12 +1024,16 @@ impl IndexSink for WorkerSink { } let ticket = SearchTicket(guard.next_ticket); guard.next_ticket += 1; - guard.searches.push_back(Queued { ticket, text: text.to_owned(), top_k, due: Instant::now() + SEARCH_DEADLINE }); + guard.searches.push_back(Queued { ticket, text: text.to_owned(), top_k, due: Instant::now() + self.deadline }); drop(guard); self.shared.1.notify_all(); Ok(ticket) } + fn search_deadline(&self) -> Duration { + self.deadline + } + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { std::mem::take(&mut lock(&self.shared).found) } diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index 76c3ddd..17ead91 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -19,6 +19,20 @@ use super::*; use crate::daemon::Daemon; use crate::daemon::tests::{INSTANCE, OTHER, batch, campaign_seed, now, question_and_ruling, slug}; +thread_local! { + /// Runs once, in the worker's own turn, between the inbox being absorbed + /// and a search being taken, holding the guard `serve_search` holds. + static AFTER_ABSORB: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; +} + +/// The hook `serve_search` calls at the point where a second lock once left a +/// gap. It is the test's only way in: nothing else can reach that point. +pub(super) fn after_absorb(guard: &mut Shared) { + if let Some(hook) = AFTER_ABSORB.with(|hook| hook.borrow_mut().take()) { + hook(guard); + } +} + fn collection() -> String { collection_name(&slug(INSTANCE)) } @@ -84,6 +98,17 @@ fn projected(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) projector } +/// A projector whose collection was lost while it ran and has been made again: +/// it owes every entry, and is rebuilding. +fn rebuilding(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { + let mut projector = projected(embedder, index, &entries[..1]); + index.state.lock().unwrap().collections.remove(&collection()); + projector.want(entries[1..].iter().cloned()); + projector.advance().expect_err("the write meets no collection"); + assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the reconciliation makes the collection again"); + projector +} + /// P4's function, as voidbot's `toQdrantPointId` computes it: the first sixteen /// bytes of the SHA-256, version and variant bits set. The fixed vectors were /// computed independently of this crate. @@ -347,8 +372,7 @@ fn every_call_on_the_search_path_is_bounded_by_the_searchs_time_and_the_flush_by .chain(index.bounds.iter().map(|(_, bound)| *bound)) .collect(); assert!(bounds.iter().all(|bound| !bound.is_zero() && *bound <= within), "{bounds:?}"); - let served = crate::serve::ServeOptions::default().search_timeout; - assert_eq!(served, SEARCH_DEADLINE); + assert_eq!(Backoff::default().search_deadline, SEARCH_DEADLINE); assert!(SEARCH_DEADLINE < BULK_EMBED_TIMEOUT); } @@ -435,7 +459,7 @@ fn committed(response: HuginnMindResponse) { } fn quick() -> Backoff { - Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600) } + Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) } } /// The index never delays or refuses admission: the embedder is shut, the @@ -811,7 +835,7 @@ fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() { seen.push(delay.as_secs()); } assert_eq!(seen, [30, 60, 120, 240, 480, 600, 600, 600]); - let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3), recheck: Duration::from_secs(3600) }; + let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) }; assert_eq!(small.after(Duration::from_millis(1)), Duration::from_millis(2)); assert_eq!(small.after(Duration::from_millis(2)), Duration::from_millis(3)); } @@ -910,7 +934,7 @@ fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { assert_eq!(projector.progress_status(), IndexStatus::Current); assert_eq!(projector.advance().unwrap(), Advance::Idle); let identified = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - assert_eq!(identified.progress_status(), IndexStatus::Reconciling { pending: 0 }, "an identified mind has yet to reconcile"); + assert_eq!(identified.progress_status(), IndexStatus::Behind { pending: 0 }, "an identified mind has yet to build its first index: lag, not a rebuild"); assert!(index.state.lock().unwrap().collections.is_empty()); let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); @@ -1037,7 +1061,7 @@ fn a_search_refuses_when_the_index_is_not_reconciled_with_the_model() { let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - assert!(format!("{:#}", fresh.search("anything", 3, SEARCH_DEADLINE).unwrap_err()).contains("being rebuilt")); + assert!(format!("{:#}", fresh.search("anything", 3, SEARCH_DEADLINE).unwrap_err()).contains("not been built")); assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); let mut projector = projected(&embedder, &index, &entries); @@ -1238,7 +1262,7 @@ fn a_search_is_refused_only_for_a_rebuild_or_a_failure() { let sink_in = |status: S, inbox: usize| { let shared = shared_with(entries_of(inbox), None, Vec::new(), false); lock(&shared).status = status; - WorkerSink { shared, identified: true } + WorkerSink { shared, identified: true, deadline: SEARCH_DEADLINE } }; assert!(ask_sink(&mut sink_in(S::Current, 2), "anything", 3).is_ok(), "admitted a moment ago and not yet absorbed"); assert!(ask_sink(&mut sink_in(S::Behind { pending: 4 }, 0), "anything", 3).is_ok(), "the worker is writing"); @@ -1251,10 +1275,12 @@ fn a_search_is_refused_only_for_a_rebuild_or_a_failure() { let (_root, mind) = mind_with_documents(); let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); - let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - fresh.want(entries.iter().cloned()); - let refusal = format!("{:#}", fresh.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); + let (lost_embedder, lost_index) = pair(); + let mut rebuilding_now = rebuilding(&lost_embedder, &lost_index, &entries); + let searches = lost_index.state.lock().unwrap().searches.len(); + let refusal = format!("{:#}", rebuilding_now.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); + assert_eq!(lost_index.state.lock().unwrap().searches.len(), searches, "the rebuilt collection was not asked"); assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); // Lag it lets through: catching up is the worker's, before the search is taken. @@ -1524,9 +1550,9 @@ fn the_worker_absorbs_admissions_before_a_search_and_waits_for_lag_but_not_for_a assert_eq!(index.state.lock().unwrap().held_when_searched, [4]); drop(guard); - // A rebuild: the projector has forgotten its reconciliation with the entries owed. - let mut rebuilding = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - rebuilding.want(entries.iter().cloned()); + // A rebuild: the collection was lost and made again, and every entry is owed. + let (rebuild_embedder, rebuild_index) = pair(); + let mut rebuilding = rebuilding(&rebuild_embedder, &rebuild_index, &entries); let shared = shared_with(Vec::new(), None, vec![queued(2, "b")], false); serve_search(&mut rebuilding, &shared); let guard = lock(&shared); @@ -1571,7 +1597,7 @@ fn a_failure_report_carries_the_pending_count_it_has_now() { assert_eq!(guard.status, IndexStatus::Failing { pending: 4, attempts: 3, error: "down".into() }); } assert!(absorb(&mut projector, &shared, None)); - assert_eq!(lock(&shared).status, IndexStatus::Reconciling { pending: 4 }); + assert_eq!(lock(&shared).status, IndexStatus::Behind { pending: 4 }, "a first build is lag"); lock(&shared).closed = true; assert!(!absorb(&mut projector, &shared, None), "a closed sink ends the worker"); } @@ -1640,3 +1666,241 @@ fn an_idle_worker_rechecks_once_per_interval() { let listed = index.state.lock().unwrap().listed; assert!((2..=40).contains(&listed), "the startup reconciliation and about five rechecks listed the collection {listed} times"); } + +// The search's rulings: what may be waiting ahead of it, when a rebuild +// refuses, what a first build is, how time is spent, and who owns the deadline. + +/// A search is never taken while a document sits in the inbox. An admission +/// that lands after the inbox was absorbed and before the search was taken (the +/// hook puts one there, where a second lock once left the gap) leaves the +/// search queued; the next turn absorbs it, and the search is answered only +/// once it is written. +#[test] +fn a_search_is_never_taken_while_an_admission_sits_in_the_inbox() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + let late = entries[3].clone(); + AFTER_ABSORB.with(|hook| *hook.borrow_mut() = Some(Box::new(move |guard| guard.inbox.push(late)))); + + serve_search(&mut projector, &shared); + { + let guard = lock(&shared); + assert_eq!(guard.searches.len(), 1, "the search was not taken past the admission"); + assert!(guard.found.is_empty()); + assert_eq!(guard.inbox.len(), 1); + } + serve_search(&mut projector, &shared); + assert_eq!(lock(&shared).searches.len(), 1, "absorbed, but not yet written"); + drain(&mut projector); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); + assert_eq!(index.state.lock().unwrap().held_when_searched, [4], "the search saw the admission"); +} + +/// Through the sink, an admission acknowledged before the search was asked is +/// in the inbox while the worker is inside a batch; the search reads it. +#[test] +fn a_search_reads_an_admission_acknowledged_before_it_was_asked() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(2), quick()); + gate.wait_arrived(1); + let admitted = entries_of(5).split_off(2); + lock(&sink.shared).inbox.extend(admitted); + let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); + gate.open(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, ticket); + assert!(answered[0].1.is_ok(), "{answered:?}"); + assert_eq!(index.state.lock().unwrap().held_when_searched, [5]); +} + +/// A collection made again is a rebuild until every document it owes is +/// written again: searches are refused as rebuilding, however the collection +/// came to be made again, and however many steps or failures the refill takes. +#[test] +fn a_collection_made_again_refuses_searches_until_it_is_refilled() { + let refused = |projector: &mut Projector, pending: u32| { + let why = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); + assert!(why.contains("being rebuilt") && why.contains(&format!("pending: {pending}")), "{why}"); + }; + let entries = entries_of(BATCH + 8); + + // Lost while the worker ran. + let (embedder, index) = pair(); + let mut projector = rebuilding(&embedder, &index, &entries); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); + refused(&mut projector, 40); + index.state.lock().unwrap().down = true; + projector.advance().expect_err("the store is down mid-refill"); + index.state.lock().unwrap().down = false; + assert_eq!(projector.advance().unwrap(), Advance::Progressed, "reconciled again, under a label that already fits"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }, "a failure does not end the rebuild"); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }, "half refilled is still a rebuild"); + refused(&mut projector, 8); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_ok(), "answered once refilled"); + assert_eq!(index.state.lock().unwrap().held_when_searched, [40]); + projector.want(entries_of(1).into_iter().map(|mut entry| { + entry.id.id.0.push_str("-later"); + entry + })); + assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 1 }, "a later admission is lag, not a rebuild"); + + // Made again because the model changed. + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + assert_eq!(projector.recheck().unwrap(), Advance::Progressed, "the change is seen and the collection made again"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); + projector.advance().unwrap(); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }); + refused(&mut projector, 8); + drain(&mut projector); + assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_ok()); +} + +/// Through the worker: a collection lost while the worker runs is made again, +/// and a search asked during the refill is refused as rebuilding, naming how +/// many documents are pending; one asked after it is answered. +#[test] +fn a_search_asked_during_a_refill_is_refused_and_one_after_it_is_answered() { + let gate = Gate::shut(); + gate.open(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let backoff = Backoff { recheck: Duration::from_millis(50), ..quick() }; + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), backoff); + sink.wait_status(|status| *status == IndexStatus::Current); + gate.shut_again(); + index.state.lock().unwrap().collections.remove(&collection()); + gate.wait_arrived(1); + let refusal = ask_sink(&mut sink, "anything", 3).unwrap_err(); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 40"), "{refusal}"); + gate.open(); + sink.wait_status(|status| *status == IndexStatus::Current); + let ticket = ask_sink(&mut sink, "anything", 3).expect("refilled"); + let answered = answers(&mut sink, 1); + assert_eq!((answered[0].0, answered[0].1.is_ok()), (ticket, true)); + assert_eq!(index.state.lock().unwrap().held_when_searched.last(), Some(&40)); +} + +/// A mind whose index has never been built has nothing stale to protect: its +/// first build is lag, whether the search was asked before the worker had +/// looked at the first admission or after. Only a rebuild of an index that +/// exists refuses. +#[test] +fn a_minds_first_build_is_lag_whoever_looks_first() { + for asked_before_the_worker_looked in [true, false] { + let what = if asked_before_the_worker_looked { "asked first" } else { "asked after the worker looked" }; + let (embedder, index) = pair(); + let entries = entries_of(3); + let (mut projector, shared) = if asked_before_the_worker_looked { + let projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + (projector, shared_with(entries, Some(MIND.into()), Vec::new(), false)) + } else { + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries); + let shared = shared_with(Vec::new(), None, Vec::new(), false); + lock(&shared).status = projector.progress_status(); + (projector, shared) + }; + let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline: SEARCH_DEADLINE }; + let ticket = ask_sink(&mut sink, "anything", 3).unwrap_or_else(|why| panic!("{what}: {why}")); + + serve_search(&mut projector, &shared); + assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 3 }, "{what}: not yet reconciled, and lag"); + assert_eq!(lock(&shared).searches.len(), 1, "{what}: the search waits for the first reconciliation"); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + serve_search(&mut projector, &shared); + assert_eq!(lock(&shared).searches.len(), 1, "{what}: and for the first writes"); + drain(&mut projector); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(matches!(guard.found.as_slice(), [(found, Ok(_))] if *found == ticket), "{what}: {:?}", guard.found); + assert_eq!(index.state.lock().unwrap().held_when_searched, [3], "{what}"); + } + + let entries = entries_of(4); + let (embedder, index) = pair(); + projected(&embedder, &index, &entries[..3]); + let mut partial = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + partial.want(entries.iter().cloned()); + partial.advance().unwrap(); + assert_eq!(partial.progress_status(), IndexStatus::Behind { pending: 1 }, "an index found whole at start-up only lags"); + + embedder.state.lock().unwrap().identity.digest = "d2".into(); + let mut replaced = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); + replaced.want(entries); + replaced.advance().unwrap(); + assert_eq!(replaced.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index that exists and is made again is a rebuild"); +} + +/// A mind with no identity and documents owed has nothing to wait for: the +/// search is taken and refused, not left queued for a worker that would spin +/// on it. +#[test] +fn a_search_is_refused_when_documents_are_owed_to_a_mind_with_no_identity() { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + projector.want(entries_of(2)); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(guard.searches.is_empty()); + let [(SearchTicket(1), Err(why))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; + assert!(why.contains("being rebuilt") && why.contains("pending: 2"), "{why}"); + assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); +} + +/// Every call a search makes is bounded by what its time has left, not by the +/// time it was given: after a slow identity read the embed's bound is smaller +/// by what the read took, and the query's by both reads. +#[test] +fn a_later_calls_bound_shrinks_as_the_search_spends_its_time() { + let entries = entries_of(3); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + let delay = Duration::from_millis(150); + let within = Duration::from_secs(10); + { + let mut state = embedder.state.lock().unwrap(); + state.identity_delay = delay; + state.bounds.clear(); + state.identity_bounds.clear(); + } + index.state.lock().unwrap().bounds.clear(); + projector.search("anything", 3, within).unwrap(); + + let embedder = embedder.state.lock().unwrap(); + let index = index.state.lock().unwrap(); + assert!(embedder.identity_bounds[0] <= within); + assert!(embedder.bounds[0] <= within - delay, "the embed came after one slow read: {:?}", embedder.bounds); + assert!(embedder.identity_bounds[1] <= within - delay, "{:?}", embedder.identity_bounds); + let queried = index.bounds.iter().find(|(call, _)| *call == "search").unwrap().1; + assert!(queried <= within - delay * 2, "the query came after two slow reads: {queried:?}"); +} + +/// The deadline has one owner: the sink. It stamps every search with it, and +/// it is what the serve loop is told (`Daemon::search_deadline`), so the loop's +/// wait and the worker's bound are one value. +#[test] +fn the_sink_owns_the_searchs_deadline_and_tells_the_loop() { + let (_root, mind) = mind_with_documents(); + let deadline = Duration::from_secs(7); + let shared = shared_with(Vec::new(), None, Vec::new(), false); + let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline }; + let asked = Instant::now(); + ask_sink(&mut sink, "anything", 3).unwrap(); + let due = lock(&shared).searches.back().expect("no worker took it").due; + assert!(due >= asked + deadline && due <= Instant::now() + deadline, "stamped with the sink's deadline"); + assert_eq!(>::search_deadline(&sink), deadline); + assert_eq!(Daemon::new(mind, sink).search_deadline(), deadline); +} diff --git a/crates/huginn-daemon/src/semantic_tests.rs b/crates/huginn-daemon/src/semantic_tests.rs index 9a5646b..513f0e9 100644 --- a/crates/huginn-daemon/src/semantic_tests.rs +++ b/crates/huginn-daemon/src/semantic_tests.rs @@ -140,7 +140,7 @@ fn ids(response: &HuginnMindResponse) -> Vec { } fn quick() -> Backoff { - Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600) } + Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) } } /// A daemon serving a mind over a real socket, with the ports the test gave it. @@ -167,7 +167,7 @@ impl Harness { &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), mind.index_entries(None).unwrap(), - quick(), + Backoff { search_deadline: search_timeout, ..quick() }, ); let mut daemon: Daemon = Daemon::new(mind, sink); let mut hub = bind("127.0.0.1:0".parse().unwrap(), &daemon.runtime_id()).unwrap(); @@ -175,7 +175,7 @@ impl Harness { let registry = schema_registry().unwrap(); let stopping = Arc::new(AtomicBool::new(false)); let loop_stopping = Arc::clone(&stopping); - let options = ServeOptions { search_timeout, ..ServeOptions::default() }; + let options = ServeOptions::default(); let serving = std::thread::spawn(move || run(&mut daemon, &mut hub, ®istry, &loop_stopping, &options)); Self { endpoint, stopping, serving: Some(serving), embedder, index, _root: root } } diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index 5af8589..a5efc02 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -88,10 +88,6 @@ pub struct ServeOptions { pub session_timeout: Duration, pub idle_sleep: Duration, pub deferred_ttl: Duration, - /// How long a semantic query may wait for the index before it is answered - /// `Unavailable`. The worker's own calls time out on their own; this is - /// what the client is promised. - pub search_timeout: Duration, } impl Default for ServeOptions { @@ -100,7 +96,6 @@ impl Default for ServeOptions { session_timeout: Duration::from_secs(30), idle_sleep: Duration::from_millis(2), deferred_ttl: Duration::from_secs(60), - search_timeout: crate::index::SEARCH_DEADLINE, } } } @@ -449,7 +444,8 @@ struct Waiting { /// Until `stopping`: expire what timed out, resend what was not acknowledged, /// answer every frame waiting on the socket, then answer every semantic query -/// the index has finished or that has waited out `search_timeout`. A semantic +/// the index has finished or that has waited out the index's own +/// `search_deadline`. A semantic /// query is never answered inline: `answer` hands back a `Search`, the loop /// holds it beside its session and goes on serving, and the embedder and the /// vector store are the worker's to wait on. A hostile datagram and a departed @@ -497,13 +493,13 @@ pub fn run>( match answer(daemon, registry, &mut bodies, message, Utc::now()) { Routed::Reply(reply) => send(hub, &session, &reply), Routed::Search { message_id, operation, search } => { - let until = Instant::now() + options.search_timeout; + let until = Instant::now() + daemon.search_deadline(); waiting.insert(search.ticket, Waiting { session, message_id, operation, search, until }); } } served += 1; } - let finished = collect_searches(daemon, &waiting, options); + let finished = collect_searches(daemon, &waiting); let answered = !finished.is_empty(); for (ticket, found) in finished { let Some(asked) = waiting.remove(&ticket) else { continue }; @@ -525,14 +521,14 @@ pub fn run>( fn collect_searches>( daemon: &mut Daemon, waiting: &BTreeMap, - options: &ServeOptions, ) -> Vec<(SearchTicket, Result)> { + let deadline = daemon.search_deadline(); let mut collected = daemon.searched(); let now = Instant::now(); for (ticket, asked) in waiting { if asked.until <= now { daemon.abandon(*ticket); - collected.push((*ticket, Err(format!("the semantic search did not finish within {:?}", options.search_timeout)))); + collected.push((*ticket, Err(format!("the semantic search did not finish within {deadline:?}")))); } } collected From 8a9ade959d05c3c97f0f781a5dc614bc1328d9e4 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:36:51 +0200 Subject: [PATCH 022/520] Drop a racy queue peek from the hung-embedder test The test read the queued search's due moment through the sink's lock right after asking, but the worker can take the search first: it failed under the load of cargo-mutants' baseline run. The stamp is pinned without a worker in the_sink_owns_the_searchs_deadline_and_tells_the_loop, which is the test that owns it. Co-Authored-By: Claude Opus 5.5 --- crates/huginn-daemon/src/index/tests.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index 17ead91..51bf48c 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -380,7 +380,7 @@ fn every_call_on_the_search_path_is_bounded_by_the_searchs_time_and_the_flush_by /// identity read never answers, and the search is answered, and the worker /// free, when the search's own time has passed. A search that waited past its /// time in the queue is answered without asking the embedder anything. A -/// ticket's time is `SEARCH_DEADLINE` from the moment it was asked. +/// ticket's time is stamped by the sink (see `the_sink_owns_the_searchs_deadline_and_tells_the_loop`). #[test] fn a_hung_embedder_holds_the_worker_no_longer_than_the_search_that_asked() { let (_root, mind) = mind_with_documents(); @@ -389,10 +389,7 @@ fn a_hung_embedder_holds_the_worker_no_longer_than_the_search_that_asked() { let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); sink.wait_status(|status| *status == IndexStatus::Current); - let asked = Instant::now(); ask_sink(&mut sink, "the ticket's time", 3).unwrap(); - let due = lock(&sink.shared).searches.back().expect("the search is queued").due; - assert!(due >= asked + SEARCH_DEADLINE && due <= Instant::now() + SEARCH_DEADLINE, "due is SEARCH_DEADLINE after it was asked"); answers(&mut sink, 1); embedder.state.lock().unwrap().hang_identity = true; From cfd9694de0c417740d2aa4507c324287e18deb5b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:03:56 +0200 Subject: [PATCH 023/520] Pin that an index found whole at start-up was built Losing a collection the worker found whole at start-up is a rebuild. The mutation that left Build::Unbuilt in place on a matching label survived, because nothing observed the difference until the collection was lost. Co-Authored-By: Claude Opus 5.5 --- crates/huginn-daemon/src/index/tests.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index 51bf48c..e890163 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -1832,6 +1832,10 @@ fn a_minds_first_build_is_lag_whoever_looks_first() { partial.want(entries.iter().cloned()); partial.advance().unwrap(); assert_eq!(partial.progress_status(), IndexStatus::Behind { pending: 1 }, "an index found whole at start-up only lags"); + drain(&mut partial); + index.state.lock().unwrap().collections.remove(&collection()); + assert_eq!(partial.recheck().unwrap(), Advance::Progressed); + assert_eq!(partial.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index found at start-up was built: losing it is a rebuild"); embedder.state.lock().unwrap().identity.digest = "d2".into(); let mut replaced = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); From 9497001d34d206738e8c070529787bfdc20a9c21 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:59:57 +0200 Subject: [PATCH 024/520] Pin settle-at-reconcile and the first-reconcile wait in take_search Two mutants survived Soul pass 4: dropping `self.settle()` from reconcile, and dropping `!projector.stale()` from take_search. One test each. Co-Authored-By: Claude Opus 5.5 --- crates/huginn-daemon/src/index/tests.rs | 3855 ++++++++++++----------- 1 file changed, 1948 insertions(+), 1907 deletions(-) diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index e890163..d18f25b 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -1,1907 +1,1948 @@ -//! The projection's rules, against in-memory ports; the adapters' request and -//! answer shapes, against a stub that speaks just enough HTTP. No test needs a -//! live Qdrant or Ollama. - -use std::collections::BTreeSet; -use std::io::{Read, Write}; -use std::net::{TcpListener, TcpStream}; -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, IndexStatus}; -use huginn_mind::{Mind, OwnedRedbMessagePackBackingStore, PipelineAdmissionOutcome}; -use tempfile::TempDir; - -use super::fakes::{FakeEmbedder, FakeIndex, Gate, Stored, pair}; -use super::ollama::OllamaEmbedder; -use super::qdrant::QdrantIndex; -use super::*; -use crate::daemon::Daemon; -use crate::daemon::tests::{INSTANCE, OTHER, batch, campaign_seed, now, question_and_ruling, slug}; - -thread_local! { - /// Runs once, in the worker's own turn, between the inbox being absorbed - /// and a search being taken, holding the guard `serve_search` holds. - static AFTER_ABSORB: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; -} - -/// The hook `serve_search` calls at the point where a second lock once left a -/// gap. It is the test's only way in: nothing else can reach that point. -pub(super) fn after_absorb(guard: &mut Shared) { - if let Some(hook) = AFTER_ABSORB.with(|hook| hook.borrow_mut().take()) { - hook(guard); - } -} - -fn collection() -> String { - collection_name(&slug(INSTANCE)) -} - -/// The bound the adapter tests give a call. -const TIME: Duration = Duration::from_secs(30); - -/// A mind identity for tests that project entries without a mind behind them. -const MIND: &str = "mind-commit-first"; - -/// A mind of the same instance name whose first batch differs, so its genesis -/// receipt id does too. -fn other_mind_of_the_same_name() -> (TempDir, Mind) { - let root = tempfile::tempdir().unwrap(); - let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - let (question, _) = question_and_ruling(); - let mut first = campaign_seed(); - first.push(question); - let outcome = mind.admit(batch(INSTANCE, first), now()); - assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); - (root, mind) -} - -/// A real mind holding a campaign, a question, the ruling that answers it and -/// the resolution admission derived: four indexable documents, beside the -/// identity and the stewardship, which carry no text. -fn mind_with_documents() -> (TempDir, Mind) { - let root = tempfile::tempdir().unwrap(); - let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - let (question, ruling) = question_and_ruling(); - for documents in [campaign_seed(), vec![question, ruling]] { - let outcome = mind.admit(batch(INSTANCE, documents), now()); - assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); - } - (root, mind) -} - -/// How many times the collection was created and how many upserts it took. -fn counts(index: &FakeIndex) -> (usize, usize) { - let state = index.state.lock().unwrap(); - (state.recreated.len(), state.upserts.len()) -} - -/// Steps until the projector has nothing to do. A step that never settles is a -/// failure of the test, not a hang: it is bounded. -fn drain(projector: &mut Projector) { - for _ in 0..1000 { - if projector.advance().unwrap() == Advance::Idle { - return; - } - } - panic!("the projector was still stepping after 1000 steps"); -} - -fn point_ids(entries: &[IndexEntry]) -> BTreeSet { - entries.iter().map(|entry| point_id(&entry.id.id.0)).collect() -} - -fn projected(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - drain(&mut projector); - projector -} - -/// A projector whose collection was lost while it ran and has been made again: -/// it owes every entry, and is rebuilding. -fn rebuilding(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { - let mut projector = projected(embedder, index, &entries[..1]); - index.state.lock().unwrap().collections.remove(&collection()); - projector.want(entries[1..].iter().cloned()); - projector.advance().expect_err("the write meets no collection"); - assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the reconciliation makes the collection again"); - projector -} - -/// P4's function, as voidbot's `toQdrantPointId` computes it: the first sixteen -/// bytes of the SHA-256, version and variant bits set. The fixed vectors were -/// computed independently of this crate. -#[test] -fn point_ids_are_a_pure_function_of_the_document_id() { - assert_eq!(point_id("eureka-state:question:Q1"), "1e8a661b-4b7e-5245-a9ad-b15a3fa27031"); - assert_eq!(point_id("eureka-state:campaign:self"), "d4135998-051d-57fb-9360-82964874fbc9"); - assert_eq!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q1")); - assert_ne!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q2")); -} - -/// The projection converges by set difference: after a crash that lost two -/// points, a restart embeds and writes those two, and nothing else, and the -/// collection is not rebuilt. -#[test] -fn reconcile_indexes_exactly_the_indexable_documents_the_collection_lacks() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - assert_eq!(entries.len(), 4); - - let (embedder, index) = pair(); - projected(&embedder, &index, &entries); - assert_eq!( - index.holding(&collection()), - point_ids(&entries), - "one point per indexable document, none for the identity or the stewardship" - ); - - let lost: Vec = entries.iter().take(2).cloned().collect(); - { - let mut state = index.state.lock().unwrap(); - let stored = state.collections.get_mut(&collection()).unwrap(); - for entry in &lost { - stored.points.remove(&point_id(&entry.id.id.0)); - } - state.upserts.clear(); - state.recreated.clear(); - } - - let restarted = FakeEmbedder::new("d1"); - projected(&restarted, &index, &entries); - let state = index.state.lock().unwrap(); - assert!(state.recreated.is_empty(), "a collection that agrees is not rebuilt"); - let written: BTreeSet = state.upserts.concat().into_iter().collect(); - assert_eq!(written, point_ids(&lost), "exactly the missing points were written"); - assert_eq!(state.upserts.concat().len(), 2, "each once"); - assert_eq!(restarted.texts().len(), 2, "and only their text was embedded"); - drop(state); - assert_eq!(index.holding(&collection()), point_ids(&entries)); -} - -/// Compatibility is the model by name and digest, its dimensions and the text -/// version. Any of them differing rebuilds the collection whole, and no vector -/// of one model survives beside another's. -#[test] -fn a_model_digest_change_rebuilds_the_collection() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let changes: [(&str, Box); 3] = [ - ("digest", Box::new(|identity| identity.digest = "d2".into())), - ("name", Box::new(|identity| identity.name = "other-model".into())), - ("dimensions", Box::new(|identity| identity.dimensions = 8)), - ]; - for (what, change) in changes { - let (embedder, index) = pair(); - projected(&embedder, &index, &entries); - assert_eq!(index.state.lock().unwrap().recreated.len(), 1, "the first run creates the collection"); - - let second = FakeEmbedder::new("d1"); - change(&mut second.state.lock().unwrap().identity); - let expected = second.state.lock().unwrap().identity.clone(); - projected(&second, &index, &entries); - let state = index.state.lock().unwrap(); - assert_eq!(state.recreated.len(), 2, "a changed {what} rebuilds"); - let (_, meta) = state.recreated.last().unwrap(); - assert_eq!( - (&meta.model, &meta.model_digest, meta.dimensions), - (&expected.name, &expected.digest, expected.dimensions), - "{what}" - ); - assert_eq!(second.texts().len(), 4, "every document is embedded again under {what}"); - assert_eq!(state.collections[&collection()].points.len(), 4); - } - - // The same model, digest and dimensions: nothing is rebuilt or rewritten. - let (embedder, index) = pair(); - projected(&embedder, &index, &entries); - let same = FakeEmbedder::new("d1"); - projected(&same, &index, &entries); - assert_eq!(index.state.lock().unwrap().recreated.len(), 1); - assert!(same.texts().is_empty()); - - // A text version the collection was not built with is a rebuild too. - { - let mut state = index.state.lock().unwrap(); - let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; - meta.index_text_version += 1; - } - projected(&FakeEmbedder::new("d1"), &index, &entries); - assert_eq!(index.state.lock().unwrap().recreated.len(), 2); -} - -/// No cross-mind writes: a collection that another writer or another instance -/// made, or that carries no label, is left exactly as it was. -#[test] -fn a_collection_owned_by_another_instance_is_never_recreated() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let ours = |managed_by: &str, instance: &str, mind: &str| CollectionMeta { - managed_by: managed_by.into(), - instance: instance.into(), - mind: mind.into(), - model: "model".into(), - model_digest: "d1".into(), - dimensions: 4, - index_text_version: INDEX_TEXT_VERSION, - }; - for foreign in [ - Described::Unlabelled, - Described::Labelled(ours(MANAGED_BY, OTHER, MIND)), - Described::Labelled(ours("voidbot", INSTANCE, MIND)), - Described::Labelled(ours(MANAGED_BY, INSTANCE, "mind-commit-another")), - Described::Labelled(ours(MANAGED_BY, INSTANCE, "")), - ] { - let (embedder, index) = pair(); - index.plant(&collection(), foreign.clone()); - let sentinel = point_id("someone-elses-document"); - index.state.lock().unwrap().collections.get_mut(&collection()).unwrap().points.insert( - sentinel.clone(), - Point { - id: sentinel.clone(), - vector: vec![0.0; 4], - payload: PointPayload { - doc_id: "someone-elses-document".into(), - kind: "x".into(), - root: "x".into(), - ordinal: 1, - text_sha256: "x".into(), - }, - }, - ); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - let error = projector.reconcile().expect_err("a foreign collection is refused"); - assert!(matches!(error, StepError::Refused(_)), "{foreign:?}: {error}"); - assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Refused { pending: 4, .. })); - let state = index.state.lock().unwrap(); - assert!(state.recreated.is_empty() && state.upserts.is_empty(), "{foreign:?} was written to"); - assert_eq!(state.collections[&collection()].label, foreign); - assert!(state.collections[&collection()].points.contains_key(&sentinel), "{foreign:?} lost a point"); - assert!(embedder.texts().is_empty()); - } -} - -/// The vector's length is checked against the collection's, so a flush that -/// the model did not change is still refused when the embedder answers wrongly. -#[test] -fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - projector.reconcile().unwrap(); - embedder.state.lock().unwrap().vector_length = Some(8); - let error = projector.flush_batch().expect_err("a longer vector is refused"); - assert!(matches!(error, StepError::Unavailable(_)), "{error}"); - assert_eq!(projector.pending(), 4); - assert!(index.state.lock().unwrap().upserts.is_empty()); -} - -/// A flush embeds through the same check as a search: a model that changed -/// before the embed is never asked, a model that changed during it has its -/// vectors dropped, and a collection that is no longer this mind's is refused. -/// In each case nothing is written, the documents stay wanted, and the -/// reconciliation is forgotten. -#[test] -fn a_flush_is_refused_as_a_search_is_when_the_model_or_the_collection_changed() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let reconciled = || { - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - projector.reconcile().unwrap(); - (embedder, index, projector) - }; - - let (embedder, index, mut projector) = reconciled(); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - projector.flush_batch().expect("a changed model is not a failure"); - assert!(embedder.texts().is_empty(), "before: the texts were not embedded under the new model"); - assert!(index.state.lock().unwrap().upserts.is_empty()); - assert_eq!((projector.pending(), projector.stale()), (4, true), "before"); - - let (embedder, index, mut projector) = reconciled(); - change_label(&index, |meta| meta.mind = "mind-commit-another".into()); - let error = projector.flush_batch().expect_err("another mind's label is refused"); - assert!(error.to_string().contains("no longer the one this mind's index was built as"), "{error}"); - assert!(embedder.texts().is_empty(), "label: nothing was embedded"); - assert!(index.state.lock().unwrap().upserts.is_empty()); - assert_eq!((projector.pending(), projector.stale()), (4, true), "label"); - - let gate = Gate::shut(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let index = FakeIndex::default(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries.iter().cloned()); - projector.reconcile().unwrap(); - let flushing = std::thread::spawn(move || { - let flushed = projector.flush_batch(); - (projector, flushed) - }); - gate.wait_arrived(1); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - gate.open(); - let (projector, flushed) = flushing.join().unwrap(); - flushed.expect("a changed model is not a failure"); - assert!(index.state.lock().unwrap().upserts.is_empty(), "during: the vectors were not written"); - assert_eq!((projector.pending(), projector.stale()), (4, true), "during"); -} - -fn change_label(index: &FakeIndex, change: impl FnOnce(&mut CollectionMeta)) { - let mut state = index.state.lock().unwrap(); - let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; - change(meta); -} -/// Every call on the search path is bounded by what remains of the search's own -/// time: the identity reads, the label read, the embed and the query. A search -/// whose embed cannot be recalled therefore cannot hold the writes off past the -/// moment nobody is waiting for it. A bulk flush keeps its own, longer, bound. -#[test] -fn every_call_on_the_search_path_is_bounded_by_the_searchs_time_and_the_flush_by_its_own() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - { - let state = embedder.state.lock().unwrap(); - assert!(!state.bounds.is_empty()); - let near = |bound: &Duration| *bound <= BULK_EMBED_TIMEOUT && *bound > BULK_EMBED_TIMEOUT - Duration::from_secs(60); - assert!(state.bounds.iter().all(near), "the flushes' embeds: {:?}", state.bounds); - assert!(state.identity_bounds.iter().all(near), "the reconciliation's and the flushes' identity reads: {:?}", state.identity_bounds); - } - embedder.state.lock().unwrap().bounds.clear(); - embedder.state.lock().unwrap().identity_bounds.clear(); - index.state.lock().unwrap().bounds.clear(); - - let within = Duration::from_secs(20); - projector.search("anything", 3, within).unwrap(); - let embedder = embedder.state.lock().unwrap(); - let index = index.state.lock().unwrap(); - assert_eq!(embedder.identity_bounds.len(), 2, "the model is read before and after the embed"); - assert_eq!(embedder.bounds.len(), 1); - assert_eq!(index.bounds.iter().map(|(call, _)| *call).collect::>(), ["describe", "search"]); - let bounds: Vec = embedder - .identity_bounds - .iter() - .chain(&embedder.bounds) - .copied() - .chain(index.bounds.iter().map(|(_, bound)| *bound)) - .collect(); - assert!(bounds.iter().all(|bound| !bound.is_zero() && *bound <= within), "{bounds:?}"); - assert_eq!(Backoff::default().search_deadline, SEARCH_DEADLINE); - assert!(SEARCH_DEADLINE < BULK_EMBED_TIMEOUT); -} - -/// A hung Ollama does not hold the worker past the search that asked: the -/// identity read never answers, and the search is answered, and the worker -/// free, when the search's own time has passed. A search that waited past its -/// time in the queue is answered without asking the embedder anything. A -/// ticket's time is stamped by the sink (see `the_sink_owns_the_searchs_deadline_and_tells_the_loop`). -#[test] -fn a_hung_embedder_holds_the_worker_no_longer_than_the_search_that_asked() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - - ask_sink(&mut sink, "the ticket's time", 3).unwrap(); - answers(&mut sink, 1); - - embedder.state.lock().unwrap().hang_identity = true; - let started = Instant::now(); - let due = started + Duration::from_millis(400); - lock(&sink.shared).searches.push_back(Queued { ticket: SearchTicket(70), text: "hung".into(), top_k: 3, due }); - sink.shared.1.notify_all(); - let answered = answers(&mut sink, 1); - assert_eq!(answered[0].0, SearchTicket(70)); - assert!(answered[0].1.is_err(), "{answered:?}"); - assert!(started.elapsed() < Duration::from_secs(5), "the worker was held for {:?}", started.elapsed()); - - embedder.state.lock().unwrap().hang_identity = false; - let reads = embedder.state.lock().unwrap().identity_bounds.len(); - lock(&sink.shared).searches.push_back(Queued { - ticket: SearchTicket(71), - text: "late".into(), - top_k: 3, - due: Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now), - }); - sink.shared.1.notify_all(); - let late = answers(&mut sink, 1); - let Err(why) = &late[0].1 else { panic!("{late:?}") }; - assert!(why.contains("has run out"), "{why}"); - assert_eq!(embedder.state.lock().unwrap().identity_bounds.len(), reads, "nothing was asked for a search nobody waits for"); - - let ticket = ask_sink(&mut sink, "the worker is free", 3).expect("the worker is still current"); - assert_eq!(answers(&mut sink, 1)[0].0, ticket); -} - -/// The Ollama adapter enforces the bound it is given: an embed that is never -/// answered fails at the bound, not at the adapter's default. -#[test] -fn an_ollama_embed_that_is_never_answered_fails_at_its_bound() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - let held = std::thread::spawn(move || { - let mut held = Vec::new(); - for stream in listener.incoming().take(1) { - held.push(stream.unwrap()); - std::thread::sleep(Duration::from_secs(8)); - } - }); - let (sent, received) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let mut embedder = OllamaEmbedder::new(&url, "m:1"); - sent.send(embedder.embed(&["a".into()], Duration::from_millis(300)).is_err()).ok(); - }); - assert_eq!(received.recv_timeout(Duration::from_secs(5)), Ok(true), "the embed did not fail at its bound"); - drop(held); -} -fn whoami_index(daemon: &mut Daemon) -> IndexStatus { - match daemon.handle(HuginnMindRequest::Whoami, now()).answered() { - HuginnMindResponse::Whoami(status) => status.index, - other => panic!("expected a status, got {other:?}"), - } -} - -fn committed(response: HuginnMindResponse) { - assert!( - matches!(response, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), - "{response:?}" - ); -} - -fn quick() -> Backoff { - Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) } -} - -/// The index never delays or refuses admission: the embedder is shut, the -/// worker is inside it, and two more admissions are answered at once. The gate -/// counts arrivals, so this observes where the work is and not how long it -/// took. -#[test] -fn an_admission_replies_while_the_embedder_is_blocked() { - let root = tempfile::tempdir().unwrap(); - let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - let gate = Gate::shut(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let index = FakeIndex::default(); - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), mind.index_entries(None).unwrap(), quick()); - let mut daemon = Daemon::new(mind, sink); - daemon.index().wait_status(|status| *status == IndexStatus::Current); - - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, campaign_seed())), now()).answered()); - gate.wait_arrived(1); - let (question, ruling) = question_and_ruling(); - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()).answered()); - assert_eq!( - whoami_index(&mut daemon), - IndexStatus::Behind { pending: 4 }, - "the campaign is in the embedder and three more are queued" - ); - assert!(index.holding(&collection()).is_empty(), "nothing has been written yet"); - - gate.open(); - daemon.index().wait_status(|status| *status == IndexStatus::Current); - assert_eq!(index.holding(&collection()).len(), 4); - assert_eq!(whoami_index(&mut daemon), IndexStatus::Current); -} - -/// A failing embedder is reported by `whoami`, retried on the backoff, and -/// nothing is dropped: when it returns, every document is written. -#[test] -fn a_failing_embedder_is_visible_in_whoami_and_retried_never_dropped() { - let root = tempfile::tempdir().unwrap(); - let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); - let mut daemon = Daemon::new(mind, sink); - daemon.index().wait_status(|status| *status == IndexStatus::Current); - - embedder.state.lock().unwrap().down = true; - let (question, ruling) = question_and_ruling(); - for documents in [campaign_seed(), vec![question, ruling]] { - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), now()).answered()); - } - let seen = daemon.index().wait_status(|status| matches!(status, IndexStatus::Failing { attempts, .. } if *attempts >= 2)); - let IndexStatus::Failing { pending, error, .. } = seen else { panic!() }; - assert_eq!(pending, 4); - assert!(error.contains("the embedder is down"), "{error}"); - assert!(matches!(whoami_index(&mut daemon), IndexStatus::Failing { .. })); - assert!(index.holding(&collection()).is_empty()); - - embedder.state.lock().unwrap().down = false; - daemon.index().wait_status(|status| *status == IndexStatus::Current); - assert_eq!(index.holding(&collection()).len(), 4, "every admitted document was kept and written"); -} - -/// A collection that is not this mind's is reported as refused, never -/// recreated, and re-read on the backoff: once it is gone the projection -/// completes. -#[test] -fn a_refused_collection_is_reported_and_reread() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - index.plant(&collection(), Described::Unlabelled); - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries, quick()); - let mut daemon = Daemon::new(mind, sink); - daemon.index().wait_status(|status| matches!(status, IndexStatus::Refused { attempts, .. } if *attempts >= 2)); - assert!(matches!(whoami_index(&mut daemon), IndexStatus::Refused { pending: 4, .. })); - assert!(index.state.lock().unwrap().recreated.is_empty()); - - index.state.lock().unwrap().collections.remove(&collection()); - daemon.index().wait_status(|status| *status == IndexStatus::Current); - assert_eq!(index.holding(&collection()).len(), 4); -} - -// The adapters, against a stub that answers the routes it is given. - -struct Stub { - url: String, - seen: Arc>>, -} - -type Route = (&'static str, u16, &'static str); - -fn find(haystack: &[u8], needle: &[u8]) -> Option { - haystack.windows(needle.len()).position(|window| window == needle) -} - -/// Serves each route once, in order, then repeats the last route that matched. -/// A route matches `" "` by prefix. -fn stub(routes: &[Route]) -> Stub { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let url = format!("http://{}", listener.local_addr().unwrap()); - let seen = Arc::new(Mutex::new(Vec::new())); - let routes: Arc>> = Arc::new(Mutex::new(routes.iter().map(|route| (*route, false)).collect())); - let log = Arc::clone(&seen); - std::thread::spawn(move || { - for stream in listener.incoming() { - let Ok(stream) = stream else { break }; - let (routes, log) = (Arc::clone(&routes), Arc::clone(&log)); - std::thread::spawn(move || answer_connection(stream, &routes, &log)); - } - }); - Stub { url, seen } -} - -fn answer_connection( - mut stream: TcpStream, - routes: &Mutex>, - log: &Mutex>, -) { - let mut buffer: Vec = Vec::new(); - loop { - let header_end = loop { - if let Some(at) = find(&buffer, b"\r\n\r\n") { - break at + 4; - } - let mut chunk = [0_u8; 4096]; - match stream.read(&mut chunk) { - Ok(0) | Err(_) => return, - Ok(read) => buffer.extend_from_slice(&chunk[..read]), - } - }; - let head = String::from_utf8_lossy(&buffer[..header_end]).into_owned(); - let length = head - .lines() - .find_map(|line| line.to_ascii_lowercase().strip_prefix("content-length:").map(|value| value.trim().parse::().unwrap())) - .unwrap_or(0); - while buffer.len() < header_end + length { - let mut chunk = [0_u8; 4096]; - match stream.read(&mut chunk) { - Ok(0) | Err(_) => return, - Ok(read) => buffer.extend_from_slice(&chunk[..read]), - } - } - let body = String::from_utf8_lossy(&buffer[header_end..header_end + length]).into_owned(); - buffer.drain(..header_end + length); - let mut first = head.lines().next().unwrap().split(' '); - let (method, path) = (first.next().unwrap().to_owned(), first.next().unwrap().to_owned()); - log.lock().unwrap().push((method.clone(), path.clone(), body)); - let key = format!("{method} {path}"); - let (status, answer) = { - let mut routes = routes.lock().unwrap(); - let matching: Vec = - routes.iter().enumerate().filter(|(_, ((prefix, ..), _))| key.starts_with(prefix)).map(|(at, _)| at).collect(); - let chosen = matching.iter().copied().find(|at| !routes[*at].1).or(matching.last().copied()); - match chosen { - Some(at) => { - routes[at].1 = true; - (routes[at].0.1, routes[at].0.2) - } - None => (404, "{}"), - } - }; - let response = - format!("HTTP/1.1 {status} X\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{answer}", answer.len()); - if stream.write_all(response.as_bytes()).is_err() { - return; - } - } -} - -const TAGS: &str = r#"{"models":[{"name":"other:1","model":"other:1","digest":"zzz"},{"name":"m:1","model":"m:1","digest":"ac6d"}]}"#; -const SHOW: &str = r#"{"model_info":{"general.architecture":"qwen3","qwen3.embedding_length":4}}"#; -const EMBED: &str = r#"{"embeddings":[[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0]]}"#; - -#[test] -fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { - let ollama = stub(&[("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]); - let mut embedder = OllamaEmbedder::new(&ollama.url, "m:1"); - assert_eq!(embedder.model_identity(TIME).unwrap(), ModelIdentity { name: "m:1".into(), digest: "ac6d".into(), dimensions: 4 }); - let vectors = embedder.embed(&["a".into(), "b".into(), "c".into(), "d".into()], Duration::from_secs(5)).unwrap(); - assert_eq!(vectors.len(), 4); - assert_eq!(vectors[0], vec![0.5, 1.0, 2.0, 3.0]); - let seen = ollama.seen.lock().unwrap(); - let (_, _, body) = seen.iter().find(|(_, path, _)| path == "/api/embed").unwrap(); - assert!(body.contains("\"model\":\"m:1\"") && body.contains("\"input\":[\"a\",\"b\",\"c\",\"d\"]"), "{body}"); - drop(seen); - - // A listing entry that carries only `name` is still the model. - let bare = stub(&[ - ("GET /api/tags", 200, r#"{"models":[{"name":"m:1","digest":"ac6d"}]}"#), - ("POST /api/show", 200, SHOW), - ]); - assert_eq!(OllamaEmbedder::new(&bare.url, "m:1").model_identity(TIME).unwrap().digest, "ac6d"); - - let mut absent = OllamaEmbedder::new(&ollama.url, "not-pulled:1"); - assert!(format!("{:#}", absent.model_identity(TIME).unwrap_err()).contains("does not list the model")); - let refusing = stub(&[("POST /api/embed", 500, r#"{"error":"boom"}"#)]); - assert!(OllamaEmbedder::new(&refusing.url, "m:1").embed(&["a".into()], Duration::from_secs(5)).is_err()); -} - -/// Compatibility never reads the address: the same model at another address -/// leaves the collection alone, and the same address serving another digest -/// rebuilds it. -#[test] -fn the_embedders_address_is_not_part_of_the_collections_compatibility() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let routes = [("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]; - let repulled = TAGS.replace("ac6d", "beef"); - let index = FakeIndex::default(); - - let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); - first.want(entries.iter().cloned()); - drain(&mut first); - assert_eq!(counts(&index), (1, 1)); - - // Another address, the same model and digest. - let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); - moved.want(entries.iter().cloned()); - drain(&mut moved); - assert_eq!(counts(&index), (1, 1)); - - // The same model name re-pulled: another digest. - let leaked: &'static str = Box::leak(repulled.into_boxed_str()); - let other = [("GET /api/tags", 200, leaked), routes[1], routes[2]]; - let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); - rebuilt.want(entries.iter().cloned()); - drain(&mut rebuilt); - assert_eq!(counts(&index), (2, 2)); -} - -#[test] -fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { - let name = collection(); - let absent = stub(&[("GET /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); - assert_eq!(QdrantIndex::new(&absent.url).describe(&name, TIME).unwrap(), Described::Absent); - - let unlabelled = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":null}}}"#)]); - assert_eq!(QdrantIndex::new(&unlabelled.url).describe(&name, TIME).unwrap(), Described::Unlabelled); - let voidbot = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":{"managedBy":"voidbot"}}}}"#)]); - assert_eq!(QdrantIndex::new(&voidbot.url).describe(&name, TIME).unwrap(), Described::Unlabelled); - - let labelled = stub(&[( - "GET /collections/", - 200, - r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","mind":"mind-commit-first","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, - )]); - assert_eq!( - QdrantIndex::new(&labelled.url).describe(&name, TIME).unwrap(), - Described::Labelled(CollectionMeta { - managed_by: "huginn".into(), - instance: "yggdrasil".into(), - mind: "mind-commit-first".into(), - model: "m:1".into(), - model_digest: "ac6d".into(), - dimensions: 4, - index_text_version: 1, - }) - ); - - // A scroll is followed to its last page. - let paged = stub(&[ - ("POST /collections/", 200, r#"{"result":{"points":[{"id":"a"},{"id":"b"}],"next_page_offset":"b"}}"#), - ("POST /collections/", 200, r#"{"result":{"points":[{"id":"c"}],"next_page_offset":null}}"#), - ]); - let ids = QdrantIndex::new(&paged.url).ids(&name).unwrap(); - let expected: BTreeSet = ["a", "b", "c"].map(String::from).into_iter().collect(); - assert_eq!(ids, expected); - let seen = paged.seen.lock().unwrap(); - assert!(!seen[0].2.contains("offset") && seen[1].2.contains("\"offset\":\"b\""), "{seen:?}"); - drop(seen); - - // A rebuild drops, then creates with the metadata and the vector shape. - let writing = stub(&[("DELETE /collections/", 200, "{}"), ("PUT /collections/", 200, "{}")]); - let meta = CollectionMeta { - managed_by: MANAGED_BY.into(), - instance: INSTANCE.into(), - mind: MIND.into(), - model: "m:1".into(), - model_digest: "ac6d".into(), - dimensions: 4, - index_text_version: INDEX_TEXT_VERSION, - }; - let mut store = QdrantIndex::new(&writing.url); - store.recreate(&name, &meta).unwrap(); - let point = Point { - id: point_id("d"), - vector: vec![1.0, 2.0, 3.0, 4.0], - payload: PointPayload { doc_id: "d".into(), kind: "question".into(), root: "eureka-state".into(), ordinal: 7, text_sha256: "h".into() }, - }; - store.upsert(&name, &[point]).unwrap(); - let seen = writing.seen.lock().unwrap(); - let paths: Vec = seen.iter().map(|(method, path, _)| format!("{method} {path}")).collect(); - assert_eq!( - paths, - [ - format!("DELETE /collections/{name}?wait=true"), - format!("PUT /collections/{name}?wait=true"), - format!("PUT /collections/{name}/points?wait=true"), - ] - ); - for expected in ["\"size\":4", "\"distance\":\"Cosine\"", "\"on_disk\":true", "\"managed_by\":\"huginn\"", "\"index_text_version\":1"] { - assert!(seen[1].2.contains(expected), "{expected} in {}", seen[1].2); - } - for expected in ["\"doc_id\":\"d\"", "\"kind\":\"question\"", "\"root\":\"eureka-state\"", "\"ordinal\":7", "\"text_sha256\":\"h\""] { - assert!(seen[2].2.contains(expected), "{expected} in {}", seen[2].2); - } - - let failing = stub(&[("PUT /collections/", 500, r#"{"status":{"error":"disk full"}}"#)]); - assert!(QdrantIndex::new(&failing.url).upsert(&name, &[]).is_err()); -} - -/// The collection's name is the contract with the store: one per instance, -/// `huginn_mind_`. -#[test] -fn the_collection_is_named_for_the_instance() { - assert_eq!(collection_name(&slug("yggdrasil")), "huginn_mind_yggdrasil"); - assert_eq!(collection_name(&slug("a.b")), "huginn_mind_a.b"); -} - -#[test] -fn a_step_error_says_what_stopped_it() { - assert_eq!(StepError::Refused("not ours".into()).to_string(), "not ours"); - let unavailable = StepError::Unavailable(anyhow::anyhow!("inner").context("outer")); - assert_eq!(unavailable.to_string(), "outer: inner"); -} - -/// What the sink reports is the worker's health with the inbox added to the -/// pending count, and a current index with something queued is behind. -#[test] -fn queued_entries_count_toward_every_status_that_has_a_pending_count() { - use IndexStatus as S; - assert_eq!(including(&S::Current, 0), S::Current); - assert_eq!(including(&S::Current, 3), S::Behind { pending: 3 }); - assert_eq!(including(&S::Reconciling { pending: 2 }, 3), S::Reconciling { pending: 5 }); - assert_eq!(including(&S::Behind { pending: 2 }, 3), S::Behind { pending: 5 }); - assert_eq!( - including(&S::Failing { pending: 2, attempts: 4, error: "e".into() }, 3), - S::Failing { pending: 5, attempts: 4, error: "e".into() } - ); - assert_eq!( - including(&S::Refused { pending: 2, attempts: 4, reason: "r".into() }, 3), - S::Refused { pending: 5, attempts: 4, reason: "r".into() } - ); -} - -/// Dropping the sink stops the worker: the worker's hold on the shared state -/// is released. A worker that never stopped would hold it forever. -#[test] -fn dropping_the_sink_stops_the_worker() { - let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let shared = Arc::downgrade(&sink.shared); - drop(sink); - eventually("the worker letting go of the shared state", || shared.upgrade().is_none()); -} - -/// F1. The promised curve, pure: 30 s doubling to a 10 min cap, never below -/// the first delay and never past the cap. A retry that shrank would hot-loop -/// against a dead Ollama. -#[test] -fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() { - let backoff = Backoff::default(); - assert_eq!((backoff.initial, backoff.max), (Duration::from_secs(30), Duration::from_secs(600))); - assert_eq!(backoff.recheck, Duration::from_secs(60), "an idle worker verifies the model and the collection each minute"); - let mut delay = backoff.initial; - let mut seen = vec![delay.as_secs()]; - for _ in 0..7 { - let next = backoff.after(delay); - assert!(next >= delay && next >= backoff.initial, "the delay never shrinks: {delay:?} -> {next:?}"); - assert!(next <= backoff.max, "the delay never passes the cap: {next:?}"); - delay = next; - seen.push(delay.as_secs()); - } - assert_eq!(seen, [30, 60, 120, 240, 480, 600, 600, 600]); - let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) }; - assert_eq!(small.after(Duration::from_millis(1)), Duration::from_millis(2)); - assert_eq!(small.after(Duration::from_millis(2)), Duration::from_millis(3)); -} - -/// F2. A collection lost while the daemon runs is found by the next failed -/// write, recreated and refilled from what the projector knows, without a -/// restart. -#[test] -fn a_collection_lost_after_startup_is_recreated_and_refilled() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - assert_eq!(projector.progress_status(), IndexStatus::Current); - assert_eq!(index.holding(&collection()).len(), 3); - - index.state.lock().unwrap().collections.remove(&collection()); - projector.want(entries[3..].iter().cloned()); - let error = projector.advance().expect_err("the write meets no collection"); - assert!(matches!(error, StepError::Unavailable(_)), "{error}"); - assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Failing { pending: 1, .. })); - - drain(&mut projector); - assert_eq!(index.state.lock().unwrap().recreated.len(), 2, "the collection was made again"); - assert_eq!(index.holding(&collection()), point_ids(&entries), "and holds every document, not only the new one"); - assert_eq!(projector.progress_status(), IndexStatus::Current); -} - -/// F2. A model re-pulled with other dimensions is seen before anything is -/// written: no vector of the new length is offered to the old collection, and -/// the next step rebuilds the collection to match. -#[test] -fn a_model_that_changes_dimensions_after_startup_rebuilds_the_collection() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - - embedder.state.lock().unwrap().identity.dimensions = 8; - projector.want(entries[3..].iter().cloned()); - let writes = index.state.lock().unwrap().upserts.len(); - assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the change is seen, not written through"); - assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "no vector of 8 was offered to a collection of 4"); - drain(&mut projector); - - let state = index.state.lock().unwrap(); - assert_eq!(state.recreated.len(), 2); - assert_eq!(state.recreated.last().unwrap().1.dimensions, 8); - let stored = &state.collections[&collection()]; - assert_eq!(stored.points.len(), 4); - assert!(stored.points.values().all(|point| point.vector.len() == 8)); -} - -/// F3. Two minds that share an instance name never share a collection: the -/// second is refused, and the first's points, label and history are untouched. -#[test] -fn a_second_mind_of_the_same_name_is_refused_and_leaves_the_first_alone() { - let (_first_root, first) = mind_with_documents(); - let (_second_root, second) = other_mind_of_the_same_name(); - let (first_id, second_id) = (first.genesis_receipt_id().unwrap().unwrap(), second.genesis_receipt_id().unwrap().unwrap()); - assert_ne!(first_id, second_id); - - let (embedder, index) = pair(); - let mut ours = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(first_id.clone())); - ours.want(first.index_entries(None).unwrap()); - drain(&mut ours); - let held = index.holding(&collection()); - assert_eq!(held.len(), 4); - - let intruder = FakeEmbedder::new("d1"); - let mut theirs = Projector::new(intruder.clone(), index.clone(), &slug(INSTANCE), Some(second_id)); - theirs.want(second.index_entries(None).unwrap()); - let error = theirs.advance().expect_err("another mind's collection is refused"); - assert!(matches!(error, StepError::Refused(_)), "{error}"); - assert!(matches!(theirs.failure_status(&error, 1), IndexStatus::Refused { .. })); - - let state = index.state.lock().unwrap(); - assert_eq!(state.recreated.len(), 1, "never rebuilt by the second mind"); - assert_eq!(state.upserts.len(), 1, "never written by the second mind"); - assert!(intruder.texts().is_empty()); - let Described::Labelled(label) = &state.collections[&collection()].label else { panic!() }; - assert_eq!(label.mind, first_id); - drop(state); - assert_eq!(index.holding(&collection()), held); -} - -/// A mind with no receipt has no identity: nothing is reconciled and no -/// collection is made, until the first admission reveals one. -#[test] -fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { - let root = tempfile::tempdir().unwrap(); - let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); - assert_eq!(mind.genesis_receipt_id().unwrap(), None); - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); - assert_eq!(projector.progress_status(), IndexStatus::Current); - assert_eq!(projector.advance().unwrap(), Advance::Idle); - let identified = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - assert_eq!(identified.progress_status(), IndexStatus::Behind { pending: 0 }, "an identified mind has yet to build its first index: lag, not a rebuild"); - assert!(index.state.lock().unwrap().collections.is_empty()); - - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); - let mut daemon = Daemon::new(mind, sink); - let seed = campaign_seed(); - let without_text = vec![seed[0].clone(), seed[1].clone()]; - committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, without_text)), now()).answered()); - for _ in 0..500 { - if index.state.lock().unwrap().collections.contains_key(&collection()) { - break; - } - std::thread::sleep(Duration::from_millis(10)); - } - let state = index.state.lock().unwrap(); - let Described::Labelled(label) = &state.collections[&collection()].label else { panic!("no collection was made") }; - assert_eq!(Some(&label.mind), daemon.mind().genesis_receipt_id().unwrap().as_ref()); -} - -/// Waits for a condition the worker will make true, without a clock in the -/// assertion: it polls for at most ten seconds and then says what never -/// happened. -fn eventually(what: &str, mut condition: impl FnMut() -> bool) { - for _ in 0..1000 { - if condition() { - return; - } - std::thread::sleep(Duration::from_millis(5)); - } - panic!("{what} never happened"); -} - -/// A model re-pulled under the same name and dimensions but a new digest must -/// not mix its vectors with the old model's. The check is before the write, -/// not after a failure: nothing is offered to the old collection, the -/// collection is rebuilt under the new digest, and every document is embedded -/// again. -#[test] -fn a_model_re_pulled_with_the_same_dimensions_never_writes_beside_the_old_vectors() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - - embedder.state.lock().unwrap().identity.digest = "d2".into(); - projector.want(entries[3..].iter().cloned()); - let writes = index.state.lock().unwrap().upserts.len(); - assert_eq!(projector.advance().unwrap(), Advance::Progressed); - assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "the new vector was not written into the old collection"); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 1 }, "the reconciliation was forgotten, not the entry"); - - drain(&mut projector); - let state = index.state.lock().unwrap(); - assert_eq!(state.recreated.len(), 2); - assert_eq!(state.recreated.last().unwrap().1.model_digest, "d2"); - let stored = &state.collections[&collection()]; - assert_eq!(stored.points.len(), 4, "every document is in the rebuilt collection"); - assert_eq!(embedder.texts().len(), 3 + 4, "and was embedded again under the new digest"); -} - -/// Nothing is written while idle, so no write can fail to announce that the -/// collection is gone or the model has changed. The idle worker looks again on -/// its own clock: a lost collection is refilled and a re-pulled model rebuilds -/// it, with no admission to prompt either. -#[test] -fn an_idle_worker_finds_a_lost_collection_and_a_changed_model_without_a_write() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let recheck = Backoff { recheck: Duration::from_millis(20), ..quick() }; - let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries.clone(), recheck); - sink.wait_status(|status| *status == IndexStatus::Current); - assert_eq!(index.holding(&collection()), point_ids(&entries)); - - index.state.lock().unwrap().collections.remove(&collection()); - eventually("the lost collection being made again and refilled", || { - index.state.lock().unwrap().recreated.len() == 2 && index.holding(&collection()) == point_ids(&entries) - }); - - embedder.state.lock().unwrap().identity.digest = "d2".into(); - eventually("the collection being rebuilt under the new digest", || { - let state = index.state.lock().unwrap(); - state.recreated.len() == 3 - && state.recreated.last().is_some_and(|(_, meta)| meta.model_digest == "d2") - && state.collections[&collection()].points.len() == 4 - }); - sink.wait_status(|status| *status == IndexStatus::Current); -} - -/// A search embeds the query behind the model's instruction, asks the index -/// with no filter for `min(top_k * 4, 200)`, and hands back candidates by id -/// and kind; a hit of a kind this organ has no name for is left out. -#[test] -fn a_search_embeds_the_query_with_its_instruction_and_oversamples() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - index.state.lock().unwrap().hits = vec![ - Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }, - Hit { doc_id: "eureka-state:mystery:X".into(), kind: "mystery".into(), score: 0.8 }, - ]; - - let hits = projector.search("who owns the state", 3, SEARCH_DEADLINE).unwrap(); - assert_eq!(hits.len(), 1, "the unknown kind is left out"); - assert_eq!(hits[0].0.id.0, "eureka-state:question:Q1"); - assert_eq!(hits[0].0.kind, PipelineKind::Question); - assert_eq!(hits[0].1, 0.9); - let embedded = embedder.texts(); - assert_eq!(embedded.last().unwrap(), &format!("Instruct: {QUERY_INSTRUCTION}\nQuery: who owns the state")); - projector.search("again", 100, SEARCH_DEADLINE).unwrap(); - let searches = index.state.lock().unwrap().searches.clone(); - assert_eq!(searches.iter().map(|(name, length, limit)| (name.as_str(), *length, *limit)).collect::>(), [ - (collection().as_str(), 4, 12), - (collection().as_str(), 4, 200), - ]); -} - -/// A search is never run against a collection that is not known to fit the -/// model: before reconciliation, and after the model has changed, it refuses -/// by saying so rather than comparing vectors of two models. -#[test] -fn a_search_refuses_when_the_index_is_not_reconciled_with_the_model() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - assert!(format!("{:#}", fresh.search("anything", 3, SEARCH_DEADLINE).unwrap_err()).contains("not been built")); - assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); - - let mut projector = projected(&embedder, &index, &entries); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - let before = embedder.texts().len(); - assert!(format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()).contains("model changed")); - assert_eq!(embedder.texts().len(), before, "the query was not embedded under the new model"); - assert!(index.state.lock().unwrap().searches.is_empty()); - assert!(projector.stale(), "and the reconciliation is to be redone"); -} - -/// The search asks for the nearest points to a vector with no filter of any -/// kind (which documents may be shown is the mind's), and reads back the -/// document id, the kind and the score, in the order Qdrant gave them. A hit -/// missing what the join needs is an error, not a guess. -#[test] -fn qdrant_search_sends_no_filter_and_reads_ids_kinds_and_scores() { - let name = collection(); - let answering = stub(&[( - "POST /collections/", - 200, - r#"{"result":[{"id":"p1","score":0.75,"payload":{"doc_id":"eureka-state:question:Q1","kind":"question"}},{"id":"p2","score":0.5,"payload":{"doc_id":"eureka-state:ruling:R1","kind":"ruling"}}]}"#, - )]); - let hits = QdrantIndex::new(&answering.url).search(&name, &[0.5, 1.0, 2.0, 3.0], 12, TIME).unwrap(); - assert_eq!( - hits, - [ - Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.75 }, - Hit { doc_id: "eureka-state:ruling:R1".into(), kind: "ruling".into(), score: 0.5 }, - ] - ); - let seen = answering.seen.lock().unwrap(); - assert_eq!((seen[0].0.as_str(), seen[0].1.as_str()), ("POST", format!("/collections/{name}/points/search").as_str())); - for expected in ["\"vector\":[0.5,1.0,2.0,3.0]", "\"limit\":12", "\"with_vector\":false"] { - assert!(seen[0].2.contains(expected), "{expected} in {}", seen[0].2); - } - assert!(!seen[0].2.contains("filter"), "no predicate is lowered into the index: {}", seen[0].2); - drop(seen); - - let nameless = stub(&[("POST /collections/", 200, r#"{"result":[{"id":"p","score":0.5,"payload":{"kind":"question"}}]}"#)]); - assert!(QdrantIndex::new(&nameless.url).search(&name, &[1.0], 1, TIME).is_err()); - let failing = stub(&[("POST /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); - assert!(QdrantIndex::new(&failing.url).search(&name, &[1.0], 1, TIME).is_err()); -} - -fn queued(ticket: u64, text: &str) -> Queued { - Queued { ticket: SearchTicket(ticket), text: text.into(), top_k: 3, due: Instant::now() + SEARCH_DEADLINE } -} - -fn shared_with( - inbox: Vec, - mind: Option, - searches: Vec, - closed: bool, -) -> Lock { - Arc::new(( - Mutex::new(Shared { - inbox, - mind, - status: IndexStatus::Current, - closed, - searches: searches.into(), - found: Vec::new(), - next_ticket: 0, - }), - Condvar::new(), - )) -} - -/// The idle wait is for work: an entry, the mind's identity, a search or the -/// sink closing ends it at once, and only a quiet worker waits out its -/// deadline. The deadline is the caller's: a deadline already past ends the -/// wait at once too, and waking for work does not extend it. -#[test] -fn an_idle_wait_ends_for_work_and_otherwise_at_its_deadline() { - let (_root, mind) = mind_with_documents(); - let entry = mind.index_entries(None).unwrap().remove(0); - let waits = |shared: &Lock| { - let started = Instant::now(); - wait_for_work(shared, Wait::Work { until: Instant::now() + Duration::from_secs(30) }); - started.elapsed() < Duration::from_secs(10) - }; - assert!(waits(&shared_with(vec![entry], None, Vec::new(), false)), "an entry"); - assert!(waits(&shared_with(Vec::new(), Some(MIND.into()), Vec::new(), false)), "the mind's identity"); - assert!(waits(&shared_with(Vec::new(), None, vec![queued(0, "q")], false)), "a search"); - assert!(waits(&shared_with(Vec::new(), None, Vec::new(), true)), "the sink closing"); - - let quiet = shared_with(Vec::new(), None, Vec::new(), false); - let started = Instant::now(); - wait_for_work(&quiet, Wait::Work { until: started + Duration::from_millis(60) }); - assert!(started.elapsed() >= Duration::from_millis(60), "a quiet worker waits for its deadline"); - let started = Instant::now(); - wait_for_work(&quiet, Wait::Work { until: started.checked_sub(Duration::from_secs(1)).unwrap_or(started) }); - assert!(started.elapsed() < Duration::from_secs(10), "a deadline already past is not waited for"); -} - -/// A backoff is the whole delay, whatever arrives: a failing index is retried -/// on its schedule. -#[test] -fn a_backoff_wait_is_the_whole_delay_whatever_arrives() { - let (_root, mind) = mind_with_documents(); - let entry = mind.index_entries(None).unwrap().remove(0); - let busy = shared_with(vec![entry], Some(MIND.into()), vec![queued(0, "q")], false); - let started = Instant::now(); - wait_for_work(&busy, Wait::Backoff(Duration::from_millis(80))); - assert!(started.elapsed() >= Duration::from_millis(80), "the delay was waited out although work was queued"); - let closed = shared_with(Vec::new(), None, Vec::new(), true); - let started = Instant::now(); - wait_for_work(&closed, Wait::Backoff(Duration::from_secs(30))); - assert!(started.elapsed() < Duration::from_secs(10), "a closed sink ends even a backoff"); -} - -/// Searches accepted while the index was healthy are answered, with the -/// reason, when the worker fails: nothing is left to wait for its deadline. -#[test] -fn queued_searches_are_answered_with_why_when_the_worker_gives_up() { - let shared = shared_with(Vec::new(), None, vec![queued(3, "a"), queued(4, "b")], false); - refuse_searches(&shared, "the index could not advance: down"); - let guard = lock(&shared); - assert!(guard.searches.is_empty()); - assert_eq!( - guard.found, - [ - (SearchTicket(3), Err("the index could not advance: down".to_string())), - (SearchTicket(4), Err("the index could not advance: down".to_string())), - ] - ); -} - -// The semantic read's index side: when a search is answered, how many wait, -// and what the worker does between them. - -type Answers = Vec<(SearchTicket, Result)>; - -/// The sink's own calls, named: the trait is generic over the store and the -/// worker does not care which. -fn ask_sink(sink: &mut WorkerSink, text: &str, top_k: u32) -> Result { - >::search(sink, text, top_k) -} - -fn collected(sink: &mut WorkerSink) -> Answers { - >::searched(sink) -} - -fn abandon(sink: &mut WorkerSink, ticket: SearchTicket) { - >::abandon(sink, ticket); -} - -/// Waits for `count` answers and returns them, in the order they were given. -fn answers(sink: &mut WorkerSink, count: usize) -> Answers { - let mut all = Vec::new(); - eventually("the searches being answered", || { - all.extend(collected(sink)); - all.len() >= count - }); - all -} - -/// `count` documents that share one document's text and differ in id, for the -/// tests whose subject is how many there are. -fn entries_of(count: usize) -> Vec { - let (_root, mind) = mind_with_documents(); - let template = mind.index_entries(None).unwrap().remove(0); - (0..count) - .map(|n| { - let mut entry = template.clone(); - entry.id.id.0 = format!("{}#{n}", entry.id.id.0); - entry - }) - .collect() -} - -fn queries(embedder: &FakeEmbedder) -> Vec { - embedder.texts().into_iter().filter(|text| text.starts_with("Instruct: ")).collect() -} - -/// A search is refused only while the index is being rebuilt or cannot be -/// asked, and the words name the state and how many documents are pending. Lag -/// is not a rebuild: the sink and the projector ask the one predicate and both -/// let it through. -#[test] -fn a_search_is_refused_only_for_a_rebuild_or_a_failure() { - use IndexStatus as S; - assert_eq!(unsearchable(&S::Current), None); - assert_eq!(unsearchable(&S::Behind { pending: 5 }), None, "lag is waited out, not refused"); - let says = |status: S, wanted: &[&str]| { - let why = unsearchable(&status).unwrap_or_else(|| panic!("{status:?} answered")); - for word in wanted { - assert!(why.contains(word), "{status:?}: {word} in {why}"); - } - }; - says(S::Reconciling { pending: 7 }, &["being rebuilt", "reconciling", "pending: 7"]); - says(S::Failing { pending: 3, attempts: 2, error: "boom".into() }, &["failing", "boom", "pending: 3"]); - says(S::Refused { pending: 4, attempts: 2, reason: "not ours".into() }, &["refused", "not ours", "pending: 4"]); - - // The sink: what the worker last published, and the documents admitted - // since, which are lag whatever the published state says of a current index. - let sink_in = |status: S, inbox: usize| { - let shared = shared_with(entries_of(inbox), None, Vec::new(), false); - lock(&shared).status = status; - WorkerSink { shared, identified: true, deadline: SEARCH_DEADLINE } - }; - assert!(ask_sink(&mut sink_in(S::Current, 2), "anything", 3).is_ok(), "admitted a moment ago and not yet absorbed"); - assert!(ask_sink(&mut sink_in(S::Behind { pending: 4 }, 0), "anything", 3).is_ok(), "the worker is writing"); - let refusal = ask_sink(&mut sink_in(S::Reconciling { pending: 4 }, 2), "anything", 3).unwrap_err(); - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 6"), "{refusal}"); - let refusal = ask_sink(&mut sink_in(S::Failing { pending: 1, attempts: 1, error: "boom".into() }, 0), "anything", 3).unwrap_err(); - assert!(refusal.contains("failing") && refusal.contains("boom"), "{refusal}"); - - // The projector: a rebuild refuses without touching the ports to find out. - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let (lost_embedder, lost_index) = pair(); - let mut rebuilding_now = rebuilding(&lost_embedder, &lost_index, &entries); - let searches = lost_index.state.lock().unwrap().searches.len(); - let refusal = format!("{:#}", rebuilding_now.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); - assert_eq!(lost_index.state.lock().unwrap().searches.len(), searches, "the rebuilt collection was not asked"); - assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); - - // Lag it lets through: catching up is the worker's, before the search is taken. - let mut lagging = projected(&embedder, &index, &entries[..3]); - lagging.want(entries[3..].iter().cloned()); - assert_eq!(lagging.progress_status(), IndexStatus::Behind { pending: 1 }); - assert!(lagging.search("anything", 3, SEARCH_DEADLINE).is_ok()); -} - -/// A search that fails for its own reasons (the vector store errs, the -/// embedder is down, an identity read times out) fails that search alone: the -/// reconciliation stands, the status stays `Current`, the collection is not -/// listed again, and the next search is asked and answered. -#[test] -fn a_failed_search_fails_that_search_alone() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - type Fault = Box; - let faults: [(&str, Fault); 3] = [ - ("the vector store fails the search", Box::new(|_, index, on| index.state.lock().unwrap().fail_search = on)), - ("the embedder is down", Box::new(|embedder, _, on| embedder.state.lock().unwrap().down = on)), - ("the identity read never answers", Box::new(|embedder, _, on| embedder.state.lock().unwrap().hang_identity = on)), - ]; - for (what, fault) in faults { - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - let listed = index.state.lock().unwrap().listed; - fault(&embedder, &index, true); - assert!(projector.search("anything", 3, Duration::from_millis(50)).is_err(), "{what}"); - assert_eq!(projector.progress_status(), IndexStatus::Current, "{what}"); - assert!(!projector.stale(), "{what}"); - fault(&embedder, &index, false); - assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_ok(), "{what}: the next search is answered"); - assert_eq!(index.state.lock().unwrap().listed, listed, "{what}: the collection was not listed again"); - } - - // In the worker the failed search is answered and nothing else moves. - let (embedder, index) = pair(); - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let listed = index.state.lock().unwrap().listed; - index.state.lock().unwrap().fail_search = true; - let ticket = ask_sink(&mut sink, "anything", 3).unwrap(); - let failed = answers(&mut sink, 1); - assert_eq!(failed[0].0, ticket); - assert!(failed[0].1.is_err()); - index.state.lock().unwrap().fail_search = false; - let next = ask_sink(&mut sink, "anything", 3).expect("the index is still current, not being rebuilt"); - let answered = answers(&mut sink, 1); - assert_eq!((answered[0].0, answered[0].1.is_ok()), (next, true)); - assert_eq!(index.state.lock().unwrap().listed, listed, "the worker did not list the collection again"); -} - -/// A model the search finds changed is a rebuild, and its refusal names the -/// state and how many documents are pending, as every other refusal does. -#[test] -fn a_model_changed_refusal_names_the_state_and_the_pending_count() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - projector.want(entries[3..].iter().cloned()); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - let refusal = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); - assert!(refusal.contains("model changed") && refusal.contains("being rebuilt") && refusal.contains("pending: 1"), "{refusal}"); -} - -/// A search asks the store what the collection says about itself: a label for -/// another mind, no label, or no collection is refused before the query is -/// embedded, and the reconciliation is to be redone. -#[test] -fn a_search_re_reads_the_collections_label() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let cases: [(&str, Box); 3] = [ - ( - "another mind's label", - Box::new(|stored| { - let Described::Labelled(meta) = &mut stored.label else { panic!() }; - meta.mind = "mind-commit-another".into(); - }), - ), - ("no label", Box::new(|stored| stored.label = Described::Unlabelled)), - ("another text version", Box::new(|stored| { - let Described::Labelled(meta) = &mut stored.label else { panic!() }; - meta.index_text_version += 1; - })), - ]; - for (what, change) in cases { - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - change(index.state.lock().unwrap().collections.get_mut(&collection()).unwrap()); - let embedded = embedder.texts().len(); - let refusal = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); - assert!(refusal.contains("no longer the one this mind's index was built as"), "{what}: {refusal}"); - assert_eq!(embedder.texts().len(), embedded, "{what}: nothing was embedded"); - assert!(index.state.lock().unwrap().searches.is_empty(), "{what}"); - assert!(projector.stale(), "{what}"); - } - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - index.state.lock().unwrap().collections.clear(); - assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_err(), "no collection"); - assert!(projector.stale()); -} - -/// The model is read again after the query is embedded: a vector made while the -/// model changed is never compared with the collection. -#[test] -fn a_model_that_changes_while_the_query_is_embedded_is_refused() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let gate = Gate::shut(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let index = FakeIndex::default(); - gate.open(); - let mut projector = projected(&embedder, &index, &entries); - gate.shut_again(); - let searching = std::thread::spawn(move || { - let found = projector.search("anything", 3, SEARCH_DEADLINE); - (projector, found) - }); - gate.wait_arrived(1); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - gate.open(); - let (projector, found) = searching.join().unwrap(); - let refusal = format!("{:#}", found.unwrap_err()); - assert!(refusal.contains("while the texts were embedded"), "{refusal}"); - assert!(index.state.lock().unwrap().searches.is_empty(), "the vector was not used"); - assert!(projector.stale()); -} - -/// Each search has a ticket of its own, and its answer comes back once, as the -/// worker gave it: nothing is added to it and nothing is left to be -/// collected twice. -#[test] -fn every_search_has_its_own_ticket_and_its_answer_comes_back_exactly_once() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - index.state.lock().unwrap().hits = - vec![Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }]; - let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let expected: Hits = vec![( - PipelineRef { kind: PipelineKind::Question, id: Short("eureka-state:question:Q1".into()) }, - 0.9, - )]; - let mut tickets = Vec::new(); - for text in ["a", "b", "c"] { - let ticket = ask_sink(&mut sink, text, 3).unwrap(); - assert_eq!(answers(&mut sink, 1), vec![(ticket, Ok(expected.clone()))]); - assert!(collected(&mut sink).is_empty(), "an answer is handed over once"); - tickets.push(ticket); - } - assert_eq!(tickets, [SearchTicket(0), SearchTicket(1), SearchTicket(2)]); -} - -/// The queue of searches is bounded and the asker who does not fit is told. -/// One search is inside the embedder and the queue is full; the next is -/// refused by name, and when the worker has answered some it takes more. -#[test] -fn the_search_queue_is_bounded_and_the_asker_is_told_when_it_is_full() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - ask_sink(&mut sink, "in flight", 3).unwrap(); - gate.wait_arrived(1); - for n in 0..SEARCHES_QUEUED_MAX { - ask_sink(&mut sink, &format!("waiting {n}"), 3).unwrap_or_else(|why| panic!("{n} of {SEARCHES_QUEUED_MAX}: {why}")); - } - let refusal = ask_sink(&mut sink, "one too many", 3).unwrap_err(); - assert!(refusal.contains(&format!("{SEARCHES_QUEUED_MAX} searches waiting")), "{refusal}"); - - gate.open(); - let answered = answers(&mut sink, SEARCHES_QUEUED_MAX + 1); - assert_eq!(answered.len(), SEARCHES_QUEUED_MAX + 1); - assert!(ask_sink(&mut sink, "room again", 3).is_ok()); -} - -/// A search the asker has stopped waiting for is taken out of the queue and -/// never embedded; the ones on either side of it are. -#[test] -fn an_abandoned_search_is_never_embedded() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let first = ask_sink(&mut sink, "first", 3).unwrap(); - gate.wait_arrived(1); - let second = ask_sink(&mut sink, "second", 3).unwrap(); - let third = ask_sink(&mut sink, "third", 3).unwrap(); - abandon(&mut sink, second); - gate.open(); - let answered = answers(&mut sink, 2); - let tickets: Vec = answered.iter().map(|(ticket, _)| *ticket).collect(); - assert_eq!(tickets, [first, third]); - let texts = queries(&embedder); - assert_eq!(texts.len(), 2, "{texts:?}"); - assert!(texts[0].ends_with("Query: first") && texts[1].ends_with("Query: third"), "{texts:?}"); -} - -/// The worker serves one queued search and returns to its writes: three -/// waiting searches leave two after one turn. -#[test] -fn one_queued_search_is_served_per_turn() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - let shared = shared_with(Vec::new(), None, vec![queued(1, "a"), queued(2, "b"), queued(3, "c")], false); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert_eq!(guard.searches.iter().map(|queued| queued.ticket).collect::>(), [SearchTicket(2), SearchTicket(3)]); - assert_eq!(guard.found.iter().map(|(ticket, _)| *ticket).collect::>(), [SearchTicket(1)]); -} - -/// A search reads its own writes. Asked while a batch is in the embedder and -/// eight more documents wait, it is accepted, waits behind them, and is -/// answered only once every one is in the collection. -#[test] -fn a_search_waits_for_the_writes_ahead_of_it_and_reads_them() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), quick()); - gate.wait_arrived(1); - let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); - gate.open(); - let answered = answers(&mut sink, 1); - assert_eq!(answered[0].0, ticket); - assert!(answered[0].1.is_ok(), "{answered:?}"); - assert_eq!(index.state.lock().unwrap().held_when_searched, [BATCH + 8], "the search saw every document admitted before it"); - sink.wait_status(|status| *status == IndexStatus::Current); -} - -/// The worker takes admissions before the search, and leaves the search queued -/// while the writes ahead of it are unfinished: a search asked with a document -/// still in the inbox absorbs it and waits, and once the index has caught up -/// it is answered. An index being rebuilt is not waited on: the search is taken -/// and refused, naming the state and the pending count. -#[test] -fn the_worker_absorbs_admissions_before_a_search_and_waits_for_lag_but_not_for_a_rebuild() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - let shared = shared_with(vec![entries[3].clone()], None, vec![queued(1, "a")], false); - - serve_search(&mut projector, &shared); - { - let guard = lock(&shared); - assert!(guard.inbox.is_empty(), "the admission ahead of the search was absorbed"); - assert_eq!(guard.searches.len(), 1, "the search waits for it to be written"); - assert!(guard.found.is_empty()); - } - assert_eq!(projector.pending(), 1); - - drain(&mut projector); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert!(guard.searches.is_empty()); - assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); - assert_eq!(index.state.lock().unwrap().held_when_searched, [4]); - drop(guard); - - // A rebuild: the collection was lost and made again, and every entry is owed. - let (rebuild_embedder, rebuild_index) = pair(); - let mut rebuilding = rebuilding(&rebuild_embedder, &rebuild_index, &entries); - let shared = shared_with(Vec::new(), None, vec![queued(2, "b")], false); - serve_search(&mut rebuilding, &shared); - let guard = lock(&shared); - let [(SearchTicket(2), Err(refusal))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); -} - -/// A search that was queued when the worker failed is answered with why, by -/// the worker's own failure path. -#[test] -fn a_search_queued_before_the_worker_fails_is_answered_with_why() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries_of(1), quick()); - gate.wait_arrived(1); - lock(&sink.shared).searches.push_back(queued(5, "anything")); - embedder.state.lock().unwrap().down = true; - gate.open(); - let answered = answers(&mut sink, 1); - assert_eq!(answered[0].0, SearchTicket(5)); - let Err(why) = &answered[0].1 else { panic!("{answered:?}") }; - assert!(why.contains("the semantic index is failing") && why.contains("pending: 1") && why.contains("the embedder is down"), "{why}"); -} - -/// While a failure is being reported, the report's pending count is the one -/// the worker holds now, published under the lock that took the entries out of -/// the inbox: never the count from before they moved. -#[test] -fn a_failure_report_carries_the_pending_count_it_has_now() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); - projector.want(entries[..2].iter().cloned()); - let shared = shared_with(entries[2..].to_vec(), None, Vec::new(), false); - let error = StepError::Unavailable(anyhow::anyhow!("down")); - assert!(absorb(&mut projector, &shared, Some((&error, 3)))); - { - let guard = lock(&shared); - assert!(guard.inbox.is_empty()); - assert_eq!(guard.status, IndexStatus::Failing { pending: 4, attempts: 3, error: "down".into() }); - } - assert!(absorb(&mut projector, &shared, None)); - assert_eq!(lock(&shared).status, IndexStatus::Behind { pending: 4 }, "a first build is lag"); - lock(&shared).closed = true; - assert!(!absorb(&mut projector, &shared, None), "a closed sink ends the worker"); -} - -/// The worker's recheck is on its own clock: searches arriving faster than the -/// interval do not keep pushing it out. -#[test] -fn steady_searches_do_not_postpone_the_workers_recheck() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let backoff = Backoff { recheck: Duration::from_millis(200), ..quick() }; - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); - sink.wait_status(|status| *status == IndexStatus::Current); - let listed = index.state.lock().unwrap().listed; - let started = Instant::now(); - while index.state.lock().unwrap().listed < listed + 2 { - assert!(started.elapsed() < Duration::from_secs(5), "the worker never rechecked while searches kept arriving"); - let _ = ask_sink(&mut sink, "anything", 3); - collected(&mut sink); - std::thread::sleep(Duration::from_millis(10)); - } -} - -/// `wait_status` gives up at one deadline, not one per wake: a health that -/// keeps changing does not keep it waiting. -#[test] -fn wait_status_gives_up_at_its_overall_deadline_however_often_the_health_changes() { - let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); - sink.wait_status(|status| *status == IndexStatus::Current); - let shared = Arc::clone(&sink.shared); - let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let ticking = Arc::clone(&stop); - let ticker = std::thread::spawn(move || { - while !ticking.load(std::sync::atomic::Ordering::Relaxed) { - shared.1.notify_all(); - std::thread::sleep(Duration::from_millis(5)); - } - }); - let (tell, told) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let gave_up = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - sink.wait_status_for(Duration::from_millis(300), |status| *status == IndexStatus::Reconciling { pending: 99 }) - })) - .is_err(); - let _ = tell.send(gave_up); - }); - let gave_up = told.recv_timeout(Duration::from_secs(10)); - stop.store(true, std::sync::atomic::Ordering::Relaxed); - ticker.join().unwrap(); - assert_eq!(gave_up, Ok(true), "the wait did not end at its deadline"); -} - -/// An idle worker verifies once per interval, not in a loop: the recheck moves -/// its own deadline when it runs. -#[test] -fn an_idle_worker_rechecks_once_per_interval() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let backoff = Backoff { recheck: Duration::from_millis(100), ..quick() }; - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); - sink.wait_status(|status| *status == IndexStatus::Current); - std::thread::sleep(Duration::from_millis(600)); - let listed = index.state.lock().unwrap().listed; - assert!((2..=40).contains(&listed), "the startup reconciliation and about five rechecks listed the collection {listed} times"); -} - -// The search's rulings: what may be waiting ahead of it, when a rebuild -// refuses, what a first build is, how time is spent, and who owns the deadline. - -/// A search is never taken while a document sits in the inbox. An admission -/// that lands after the inbox was absorbed and before the search was taken (the -/// hook puts one there, where a second lock once left the gap) leaves the -/// search queued; the next turn absorbs it, and the search is answered only -/// once it is written. -#[test] -fn a_search_is_never_taken_while_an_admission_sits_in_the_inbox() { - let (_root, mind) = mind_with_documents(); - let entries = mind.index_entries(None).unwrap(); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries[..3]); - let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); - let late = entries[3].clone(); - AFTER_ABSORB.with(|hook| *hook.borrow_mut() = Some(Box::new(move |guard| guard.inbox.push(late)))); - - serve_search(&mut projector, &shared); - { - let guard = lock(&shared); - assert_eq!(guard.searches.len(), 1, "the search was not taken past the admission"); - assert!(guard.found.is_empty()); - assert_eq!(guard.inbox.len(), 1); - } - serve_search(&mut projector, &shared); - assert_eq!(lock(&shared).searches.len(), 1, "absorbed, but not yet written"); - drain(&mut projector); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); - assert_eq!(index.state.lock().unwrap().held_when_searched, [4], "the search saw the admission"); -} - -/// Through the sink, an admission acknowledged before the search was asked is -/// in the inbox while the worker is inside a batch; the search reads it. -#[test] -fn a_search_reads_an_admission_acknowledged_before_it_was_asked() { - let gate = Gate::shut(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(2), quick()); - gate.wait_arrived(1); - let admitted = entries_of(5).split_off(2); - lock(&sink.shared).inbox.extend(admitted); - let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); - gate.open(); - let answered = answers(&mut sink, 1); - assert_eq!(answered[0].0, ticket); - assert!(answered[0].1.is_ok(), "{answered:?}"); - assert_eq!(index.state.lock().unwrap().held_when_searched, [5]); -} - -/// A collection made again is a rebuild until every document it owes is -/// written again: searches are refused as rebuilding, however the collection -/// came to be made again, and however many steps or failures the refill takes. -#[test] -fn a_collection_made_again_refuses_searches_until_it_is_refilled() { - let refused = |projector: &mut Projector, pending: u32| { - let why = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); - assert!(why.contains("being rebuilt") && why.contains(&format!("pending: {pending}")), "{why}"); - }; - let entries = entries_of(BATCH + 8); - - // Lost while the worker ran. - let (embedder, index) = pair(); - let mut projector = rebuilding(&embedder, &index, &entries); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); - refused(&mut projector, 40); - index.state.lock().unwrap().down = true; - projector.advance().expect_err("the store is down mid-refill"); - index.state.lock().unwrap().down = false; - assert_eq!(projector.advance().unwrap(), Advance::Progressed, "reconciled again, under a label that already fits"); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }, "a failure does not end the rebuild"); - assert_eq!(projector.advance().unwrap(), Advance::Progressed); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }, "half refilled is still a rebuild"); - refused(&mut projector, 8); - assert_eq!(projector.advance().unwrap(), Advance::Progressed); - assert_eq!(projector.progress_status(), IndexStatus::Current); - assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_ok(), "answered once refilled"); - assert_eq!(index.state.lock().unwrap().held_when_searched, [40]); - projector.want(entries_of(1).into_iter().map(|mut entry| { - entry.id.id.0.push_str("-later"); - entry - })); - assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 1 }, "a later admission is lag, not a rebuild"); - - // Made again because the model changed. - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - embedder.state.lock().unwrap().identity.digest = "d2".into(); - assert_eq!(projector.recheck().unwrap(), Advance::Progressed, "the change is seen and the collection made again"); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); - projector.advance().unwrap(); - assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }); - refused(&mut projector, 8); - drain(&mut projector); - assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_ok()); -} - -/// Through the worker: a collection lost while the worker runs is made again, -/// and a search asked during the refill is refused as rebuilding, naming how -/// many documents are pending; one asked after it is answered. -#[test] -fn a_search_asked_during_a_refill_is_refused_and_one_after_it_is_answered() { - let gate = Gate::shut(); - gate.open(); - let (_, index) = pair(); - let embedder = FakeEmbedder::new("d1").behind(&gate); - let backoff = Backoff { recheck: Duration::from_millis(50), ..quick() }; - let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), backoff); - sink.wait_status(|status| *status == IndexStatus::Current); - gate.shut_again(); - index.state.lock().unwrap().collections.remove(&collection()); - gate.wait_arrived(1); - let refusal = ask_sink(&mut sink, "anything", 3).unwrap_err(); - assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 40"), "{refusal}"); - gate.open(); - sink.wait_status(|status| *status == IndexStatus::Current); - let ticket = ask_sink(&mut sink, "anything", 3).expect("refilled"); - let answered = answers(&mut sink, 1); - assert_eq!((answered[0].0, answered[0].1.is_ok()), (ticket, true)); - assert_eq!(index.state.lock().unwrap().held_when_searched.last(), Some(&40)); -} - -/// A mind whose index has never been built has nothing stale to protect: its -/// first build is lag, whether the search was asked before the worker had -/// looked at the first admission or after. Only a rebuild of an index that -/// exists refuses. -#[test] -fn a_minds_first_build_is_lag_whoever_looks_first() { - for asked_before_the_worker_looked in [true, false] { - let what = if asked_before_the_worker_looked { "asked first" } else { "asked after the worker looked" }; - let (embedder, index) = pair(); - let entries = entries_of(3); - let (mut projector, shared) = if asked_before_the_worker_looked { - let projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); - (projector, shared_with(entries, Some(MIND.into()), Vec::new(), false)) - } else { - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - projector.want(entries); - let shared = shared_with(Vec::new(), None, Vec::new(), false); - lock(&shared).status = projector.progress_status(); - (projector, shared) - }; - let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline: SEARCH_DEADLINE }; - let ticket = ask_sink(&mut sink, "anything", 3).unwrap_or_else(|why| panic!("{what}: {why}")); - - serve_search(&mut projector, &shared); - assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 3 }, "{what}: not yet reconciled, and lag"); - assert_eq!(lock(&shared).searches.len(), 1, "{what}: the search waits for the first reconciliation"); - assert_eq!(projector.advance().unwrap(), Advance::Progressed); - serve_search(&mut projector, &shared); - assert_eq!(lock(&shared).searches.len(), 1, "{what}: and for the first writes"); - drain(&mut projector); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert!(matches!(guard.found.as_slice(), [(found, Ok(_))] if *found == ticket), "{what}: {:?}", guard.found); - assert_eq!(index.state.lock().unwrap().held_when_searched, [3], "{what}"); - } - - let entries = entries_of(4); - let (embedder, index) = pair(); - projected(&embedder, &index, &entries[..3]); - let mut partial = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); - partial.want(entries.iter().cloned()); - partial.advance().unwrap(); - assert_eq!(partial.progress_status(), IndexStatus::Behind { pending: 1 }, "an index found whole at start-up only lags"); - drain(&mut partial); - index.state.lock().unwrap().collections.remove(&collection()); - assert_eq!(partial.recheck().unwrap(), Advance::Progressed); - assert_eq!(partial.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index found at start-up was built: losing it is a rebuild"); - - embedder.state.lock().unwrap().identity.digest = "d2".into(); - let mut replaced = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); - replaced.want(entries); - replaced.advance().unwrap(); - assert_eq!(replaced.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index that exists and is made again is a rebuild"); -} - -/// A mind with no identity and documents owed has nothing to wait for: the -/// search is taken and refused, not left queued for a worker that would spin -/// on it. -#[test] -fn a_search_is_refused_when_documents_are_owed_to_a_mind_with_no_identity() { - let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); - projector.want(entries_of(2)); - let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); - serve_search(&mut projector, &shared); - let guard = lock(&shared); - assert!(guard.searches.is_empty()); - let [(SearchTicket(1), Err(why))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; - assert!(why.contains("being rebuilt") && why.contains("pending: 2"), "{why}"); - assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); -} - -/// Every call a search makes is bounded by what its time has left, not by the -/// time it was given: after a slow identity read the embed's bound is smaller -/// by what the read took, and the query's by both reads. -#[test] -fn a_later_calls_bound_shrinks_as_the_search_spends_its_time() { - let entries = entries_of(3); - let (embedder, index) = pair(); - let mut projector = projected(&embedder, &index, &entries); - let delay = Duration::from_millis(150); - let within = Duration::from_secs(10); - { - let mut state = embedder.state.lock().unwrap(); - state.identity_delay = delay; - state.bounds.clear(); - state.identity_bounds.clear(); - } - index.state.lock().unwrap().bounds.clear(); - projector.search("anything", 3, within).unwrap(); - - let embedder = embedder.state.lock().unwrap(); - let index = index.state.lock().unwrap(); - assert!(embedder.identity_bounds[0] <= within); - assert!(embedder.bounds[0] <= within - delay, "the embed came after one slow read: {:?}", embedder.bounds); - assert!(embedder.identity_bounds[1] <= within - delay, "{:?}", embedder.identity_bounds); - let queried = index.bounds.iter().find(|(call, _)| *call == "search").unwrap().1; - assert!(queried <= within - delay * 2, "the query came after two slow reads: {queried:?}"); -} - -/// The deadline has one owner: the sink. It stamps every search with it, and -/// it is what the serve loop is told (`Daemon::search_deadline`), so the loop's -/// wait and the worker's bound are one value. -#[test] -fn the_sink_owns_the_searchs_deadline_and_tells_the_loop() { - let (_root, mind) = mind_with_documents(); - let deadline = Duration::from_secs(7); - let shared = shared_with(Vec::new(), None, Vec::new(), false); - let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline }; - let asked = Instant::now(); - ask_sink(&mut sink, "anything", 3).unwrap(); - let due = lock(&shared).searches.back().expect("no worker took it").due; - assert!(due >= asked + deadline && due <= Instant::now() + deadline, "stamped with the sink's deadline"); - assert_eq!(>::search_deadline(&sink), deadline); - assert_eq!(Daemon::new(mind, sink).search_deadline(), deadline); -} +//! The projection's rules, against in-memory ports; the adapters' request and +//! answer shapes, against a stub that speaks just enough HTTP. No test needs a +//! live Qdrant or Ollama. + +use std::collections::BTreeSet; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, IndexStatus}; +use huginn_mind::{Mind, OwnedRedbMessagePackBackingStore, PipelineAdmissionOutcome}; +use tempfile::TempDir; + +use super::fakes::{FakeEmbedder, FakeIndex, Gate, Stored, pair}; +use super::ollama::OllamaEmbedder; +use super::qdrant::QdrantIndex; +use super::*; +use crate::daemon::Daemon; +use crate::daemon::tests::{INSTANCE, OTHER, batch, campaign_seed, now, question_and_ruling, slug}; + +thread_local! { + /// Runs once, in the worker's own turn, between the inbox being absorbed + /// and a search being taken, holding the guard `serve_search` holds. + static AFTER_ABSORB: std::cell::RefCell>> = const { std::cell::RefCell::new(None) }; +} + +/// The hook `serve_search` calls at the point where a second lock once left a +/// gap. It is the test's only way in: nothing else can reach that point. +pub(super) fn after_absorb(guard: &mut Shared) { + if let Some(hook) = AFTER_ABSORB.with(|hook| hook.borrow_mut().take()) { + hook(guard); + } +} + +fn collection() -> String { + collection_name(&slug(INSTANCE)) +} + +/// The bound the adapter tests give a call. +const TIME: Duration = Duration::from_secs(30); + +/// A mind identity for tests that project entries without a mind behind them. +const MIND: &str = "mind-commit-first"; + +/// A mind of the same instance name whose first batch differs, so its genesis +/// receipt id does too. +fn other_mind_of_the_same_name() -> (TempDir, Mind) { + let root = tempfile::tempdir().unwrap(); + let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (question, _) = question_and_ruling(); + let mut first = campaign_seed(); + first.push(question); + let outcome = mind.admit(batch(INSTANCE, first), now()); + assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); + (root, mind) +} + +/// A real mind holding a campaign, a question, the ruling that answers it and +/// the resolution admission derived: four indexable documents, beside the +/// identity and the stewardship, which carry no text. +fn mind_with_documents() -> (TempDir, Mind) { + let root = tempfile::tempdir().unwrap(); + let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (question, ruling) = question_and_ruling(); + for documents in [campaign_seed(), vec![question, ruling]] { + let outcome = mind.admit(batch(INSTANCE, documents), now()); + assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); + } + (root, mind) +} + +/// How many times the collection was created and how many upserts it took. +fn counts(index: &FakeIndex) -> (usize, usize) { + let state = index.state.lock().unwrap(); + (state.recreated.len(), state.upserts.len()) +} + +/// Steps until the projector has nothing to do. A step that never settles is a +/// failure of the test, not a hang: it is bounded. +fn drain(projector: &mut Projector) { + for _ in 0..1000 { + if projector.advance().unwrap() == Advance::Idle { + return; + } + } + panic!("the projector was still stepping after 1000 steps"); +} + +fn point_ids(entries: &[IndexEntry]) -> BTreeSet { + entries.iter().map(|entry| point_id(&entry.id.id.0)).collect() +} + +fn projected(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + drain(&mut projector); + projector +} + +/// A projector whose collection was lost while it ran and has been made again: +/// it owes every entry, and is rebuilding. +fn rebuilding(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { + let mut projector = projected(embedder, index, &entries[..1]); + index.state.lock().unwrap().collections.remove(&collection()); + projector.want(entries[1..].iter().cloned()); + projector.advance().expect_err("the write meets no collection"); + assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the reconciliation makes the collection again"); + projector +} + +/// P4's function, as voidbot's `toQdrantPointId` computes it: the first sixteen +/// bytes of the SHA-256, version and variant bits set. The fixed vectors were +/// computed independently of this crate. +#[test] +fn point_ids_are_a_pure_function_of_the_document_id() { + assert_eq!(point_id("eureka-state:question:Q1"), "1e8a661b-4b7e-5245-a9ad-b15a3fa27031"); + assert_eq!(point_id("eureka-state:campaign:self"), "d4135998-051d-57fb-9360-82964874fbc9"); + assert_eq!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q1")); + assert_ne!(point_id("eureka-state:question:Q1"), point_id("eureka-state:question:Q2")); +} + +/// The projection converges by set difference: after a crash that lost two +/// points, a restart embeds and writes those two, and nothing else, and the +/// collection is not rebuilt. +#[test] +fn reconcile_indexes_exactly_the_indexable_documents_the_collection_lacks() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + assert_eq!(entries.len(), 4); + + let (embedder, index) = pair(); + projected(&embedder, &index, &entries); + assert_eq!( + index.holding(&collection()), + point_ids(&entries), + "one point per indexable document, none for the identity or the stewardship" + ); + + let lost: Vec = entries.iter().take(2).cloned().collect(); + { + let mut state = index.state.lock().unwrap(); + let stored = state.collections.get_mut(&collection()).unwrap(); + for entry in &lost { + stored.points.remove(&point_id(&entry.id.id.0)); + } + state.upserts.clear(); + state.recreated.clear(); + } + + let restarted = FakeEmbedder::new("d1"); + projected(&restarted, &index, &entries); + let state = index.state.lock().unwrap(); + assert!(state.recreated.is_empty(), "a collection that agrees is not rebuilt"); + let written: BTreeSet = state.upserts.concat().into_iter().collect(); + assert_eq!(written, point_ids(&lost), "exactly the missing points were written"); + assert_eq!(state.upserts.concat().len(), 2, "each once"); + assert_eq!(restarted.texts().len(), 2, "and only their text was embedded"); + drop(state); + assert_eq!(index.holding(&collection()), point_ids(&entries)); +} + +/// Compatibility is the model by name and digest, its dimensions and the text +/// version. Any of them differing rebuilds the collection whole, and no vector +/// of one model survives beside another's. +#[test] +fn a_model_digest_change_rebuilds_the_collection() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let changes: [(&str, Box); 3] = [ + ("digest", Box::new(|identity| identity.digest = "d2".into())), + ("name", Box::new(|identity| identity.name = "other-model".into())), + ("dimensions", Box::new(|identity| identity.dimensions = 8)), + ]; + for (what, change) in changes { + let (embedder, index) = pair(); + projected(&embedder, &index, &entries); + assert_eq!(index.state.lock().unwrap().recreated.len(), 1, "the first run creates the collection"); + + let second = FakeEmbedder::new("d1"); + change(&mut second.state.lock().unwrap().identity); + let expected = second.state.lock().unwrap().identity.clone(); + projected(&second, &index, &entries); + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2, "a changed {what} rebuilds"); + let (_, meta) = state.recreated.last().unwrap(); + assert_eq!( + (&meta.model, &meta.model_digest, meta.dimensions), + (&expected.name, &expected.digest, expected.dimensions), + "{what}" + ); + assert_eq!(second.texts().len(), 4, "every document is embedded again under {what}"); + assert_eq!(state.collections[&collection()].points.len(), 4); + } + + // The same model, digest and dimensions: nothing is rebuilt or rewritten. + let (embedder, index) = pair(); + projected(&embedder, &index, &entries); + let same = FakeEmbedder::new("d1"); + projected(&same, &index, &entries); + assert_eq!(index.state.lock().unwrap().recreated.len(), 1); + assert!(same.texts().is_empty()); + + // A text version the collection was not built with is a rebuild too. + { + let mut state = index.state.lock().unwrap(); + let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; + meta.index_text_version += 1; + } + projected(&FakeEmbedder::new("d1"), &index, &entries); + assert_eq!(index.state.lock().unwrap().recreated.len(), 2); +} + +/// No cross-mind writes: a collection that another writer or another instance +/// made, or that carries no label, is left exactly as it was. +#[test] +fn a_collection_owned_by_another_instance_is_never_recreated() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let ours = |managed_by: &str, instance: &str, mind: &str| CollectionMeta { + managed_by: managed_by.into(), + instance: instance.into(), + mind: mind.into(), + model: "model".into(), + model_digest: "d1".into(), + dimensions: 4, + index_text_version: INDEX_TEXT_VERSION, + }; + for foreign in [ + Described::Unlabelled, + Described::Labelled(ours(MANAGED_BY, OTHER, MIND)), + Described::Labelled(ours("voidbot", INSTANCE, MIND)), + Described::Labelled(ours(MANAGED_BY, INSTANCE, "mind-commit-another")), + Described::Labelled(ours(MANAGED_BY, INSTANCE, "")), + ] { + let (embedder, index) = pair(); + index.plant(&collection(), foreign.clone()); + let sentinel = point_id("someone-elses-document"); + index.state.lock().unwrap().collections.get_mut(&collection()).unwrap().points.insert( + sentinel.clone(), + Point { + id: sentinel.clone(), + vector: vec![0.0; 4], + payload: PointPayload { + doc_id: "someone-elses-document".into(), + kind: "x".into(), + root: "x".into(), + ordinal: 1, + text_sha256: "x".into(), + }, + }, + ); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + let error = projector.reconcile().expect_err("a foreign collection is refused"); + assert!(matches!(error, StepError::Refused(_)), "{foreign:?}: {error}"); + assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Refused { pending: 4, .. })); + let state = index.state.lock().unwrap(); + assert!(state.recreated.is_empty() && state.upserts.is_empty(), "{foreign:?} was written to"); + assert_eq!(state.collections[&collection()].label, foreign); + assert!(state.collections[&collection()].points.contains_key(&sentinel), "{foreign:?} lost a point"); + assert!(embedder.texts().is_empty()); + } +} + +/// The vector's length is checked against the collection's, so a flush that +/// the model did not change is still refused when the embedder answers wrongly. +#[test] +fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + projector.reconcile().unwrap(); + embedder.state.lock().unwrap().vector_length = Some(8); + let error = projector.flush_batch().expect_err("a longer vector is refused"); + assert!(matches!(error, StepError::Unavailable(_)), "{error}"); + assert_eq!(projector.pending(), 4); + assert!(index.state.lock().unwrap().upserts.is_empty()); +} + +/// A flush embeds through the same check as a search: a model that changed +/// before the embed is never asked, a model that changed during it has its +/// vectors dropped, and a collection that is no longer this mind's is refused. +/// In each case nothing is written, the documents stay wanted, and the +/// reconciliation is forgotten. +#[test] +fn a_flush_is_refused_as_a_search_is_when_the_model_or_the_collection_changed() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let reconciled = || { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + projector.reconcile().unwrap(); + (embedder, index, projector) + }; + + let (embedder, index, mut projector) = reconciled(); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + projector.flush_batch().expect("a changed model is not a failure"); + assert!(embedder.texts().is_empty(), "before: the texts were not embedded under the new model"); + assert!(index.state.lock().unwrap().upserts.is_empty()); + assert_eq!((projector.pending(), projector.stale()), (4, true), "before"); + + let (embedder, index, mut projector) = reconciled(); + change_label(&index, |meta| meta.mind = "mind-commit-another".into()); + let error = projector.flush_batch().expect_err("another mind's label is refused"); + assert!(error.to_string().contains("no longer the one this mind's index was built as"), "{error}"); + assert!(embedder.texts().is_empty(), "label: nothing was embedded"); + assert!(index.state.lock().unwrap().upserts.is_empty()); + assert_eq!((projector.pending(), projector.stale()), (4, true), "label"); + + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let index = FakeIndex::default(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries.iter().cloned()); + projector.reconcile().unwrap(); + let flushing = std::thread::spawn(move || { + let flushed = projector.flush_batch(); + (projector, flushed) + }); + gate.wait_arrived(1); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + gate.open(); + let (projector, flushed) = flushing.join().unwrap(); + flushed.expect("a changed model is not a failure"); + assert!(index.state.lock().unwrap().upserts.is_empty(), "during: the vectors were not written"); + assert_eq!((projector.pending(), projector.stale()), (4, true), "during"); +} + +fn change_label(index: &FakeIndex, change: impl FnOnce(&mut CollectionMeta)) { + let mut state = index.state.lock().unwrap(); + let Described::Labelled(meta) = &mut state.collections.get_mut(&collection()).unwrap().label else { panic!() }; + change(meta); +} +/// Every call on the search path is bounded by what remains of the search's own +/// time: the identity reads, the label read, the embed and the query. A search +/// whose embed cannot be recalled therefore cannot hold the writes off past the +/// moment nobody is waiting for it. A bulk flush keeps its own, longer, bound. +#[test] +fn every_call_on_the_search_path_is_bounded_by_the_searchs_time_and_the_flush_by_its_own() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + { + let state = embedder.state.lock().unwrap(); + assert!(!state.bounds.is_empty()); + let near = |bound: &Duration| *bound <= BULK_EMBED_TIMEOUT && *bound > BULK_EMBED_TIMEOUT - Duration::from_secs(60); + assert!(state.bounds.iter().all(near), "the flushes' embeds: {:?}", state.bounds); + assert!(state.identity_bounds.iter().all(near), "the reconciliation's and the flushes' identity reads: {:?}", state.identity_bounds); + } + embedder.state.lock().unwrap().bounds.clear(); + embedder.state.lock().unwrap().identity_bounds.clear(); + index.state.lock().unwrap().bounds.clear(); + + let within = Duration::from_secs(20); + projector.search("anything", 3, within).unwrap(); + let embedder = embedder.state.lock().unwrap(); + let index = index.state.lock().unwrap(); + assert_eq!(embedder.identity_bounds.len(), 2, "the model is read before and after the embed"); + assert_eq!(embedder.bounds.len(), 1); + assert_eq!(index.bounds.iter().map(|(call, _)| *call).collect::>(), ["describe", "search"]); + let bounds: Vec = embedder + .identity_bounds + .iter() + .chain(&embedder.bounds) + .copied() + .chain(index.bounds.iter().map(|(_, bound)| *bound)) + .collect(); + assert!(bounds.iter().all(|bound| !bound.is_zero() && *bound <= within), "{bounds:?}"); + assert_eq!(Backoff::default().search_deadline, SEARCH_DEADLINE); + assert!(SEARCH_DEADLINE < BULK_EMBED_TIMEOUT); +} + +/// A hung Ollama does not hold the worker past the search that asked: the +/// identity read never answers, and the search is answered, and the worker +/// free, when the search's own time has passed. A search that waited past its +/// time in the queue is answered without asking the embedder anything. A +/// ticket's time is stamped by the sink (see `the_sink_owns_the_searchs_deadline_and_tells_the_loop`). +#[test] +fn a_hung_embedder_holds_the_worker_no_longer_than_the_search_that_asked() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + + ask_sink(&mut sink, "the ticket's time", 3).unwrap(); + answers(&mut sink, 1); + + embedder.state.lock().unwrap().hang_identity = true; + let started = Instant::now(); + let due = started + Duration::from_millis(400); + lock(&sink.shared).searches.push_back(Queued { ticket: SearchTicket(70), text: "hung".into(), top_k: 3, due }); + sink.shared.1.notify_all(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, SearchTicket(70)); + assert!(answered[0].1.is_err(), "{answered:?}"); + assert!(started.elapsed() < Duration::from_secs(5), "the worker was held for {:?}", started.elapsed()); + + embedder.state.lock().unwrap().hang_identity = false; + let reads = embedder.state.lock().unwrap().identity_bounds.len(); + lock(&sink.shared).searches.push_back(Queued { + ticket: SearchTicket(71), + text: "late".into(), + top_k: 3, + due: Instant::now().checked_sub(Duration::from_secs(1)).unwrap_or_else(Instant::now), + }); + sink.shared.1.notify_all(); + let late = answers(&mut sink, 1); + let Err(why) = &late[0].1 else { panic!("{late:?}") }; + assert!(why.contains("has run out"), "{why}"); + assert_eq!(embedder.state.lock().unwrap().identity_bounds.len(), reads, "nothing was asked for a search nobody waits for"); + + let ticket = ask_sink(&mut sink, "the worker is free", 3).expect("the worker is still current"); + assert_eq!(answers(&mut sink, 1)[0].0, ticket); +} + +/// The Ollama adapter enforces the bound it is given: an embed that is never +/// answered fails at the bound, not at the adapter's default. +#[test] +fn an_ollama_embed_that_is_never_answered_fails_at_its_bound() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let held = std::thread::spawn(move || { + let mut held = Vec::new(); + for stream in listener.incoming().take(1) { + held.push(stream.unwrap()); + std::thread::sleep(Duration::from_secs(8)); + } + }); + let (sent, received) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut embedder = OllamaEmbedder::new(&url, "m:1"); + sent.send(embedder.embed(&["a".into()], Duration::from_millis(300)).is_err()).ok(); + }); + assert_eq!(received.recv_timeout(Duration::from_secs(5)), Ok(true), "the embed did not fail at its bound"); + drop(held); +} +fn whoami_index(daemon: &mut Daemon) -> IndexStatus { + match daemon.handle(HuginnMindRequest::Whoami, now()).answered() { + HuginnMindResponse::Whoami(status) => status.index, + other => panic!("expected a status, got {other:?}"), + } +} + +fn committed(response: HuginnMindResponse) { + assert!( + matches!(response, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), + "{response:?}" + ); +} + +fn quick() -> Backoff { + Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(2), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) } +} + +/// The index never delays or refuses admission: the embedder is shut, the +/// worker is inside it, and two more admissions are answered at once. The gate +/// counts arrivals, so this observes where the work is and not how long it +/// took. +#[test] +fn an_admission_replies_while_the_embedder_is_blocked() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let index = FakeIndex::default(); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), mind.index_entries(None).unwrap(), quick()); + let mut daemon = Daemon::new(mind, sink); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, campaign_seed())), now()).answered()); + gate.wait_arrived(1); + let (question, ruling) = question_and_ruling(); + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, vec![question, ruling])), now()).answered()); + assert_eq!( + whoami_index(&mut daemon), + IndexStatus::Behind { pending: 4 }, + "the campaign is in the embedder and three more are queued" + ); + assert!(index.holding(&collection()).is_empty(), "nothing has been written yet"); + + gate.open(); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 4); + assert_eq!(whoami_index(&mut daemon), IndexStatus::Current); +} + +/// A failing embedder is reported by `whoami`, retried on the backoff, and +/// nothing is dropped: when it returns, every document is written. +#[test] +fn a_failing_embedder_is_visible_in_whoami_and_retried_never_dropped() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (embedder, index) = pair(); + let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); + let mut daemon = Daemon::new(mind, sink); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + + embedder.state.lock().unwrap().down = true; + let (question, ruling) = question_and_ruling(); + for documents in [campaign_seed(), vec![question, ruling]] { + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), now()).answered()); + } + let seen = daemon.index().wait_status(|status| matches!(status, IndexStatus::Failing { attempts, .. } if *attempts >= 2)); + let IndexStatus::Failing { pending, error, .. } = seen else { panic!() }; + assert_eq!(pending, 4); + assert!(error.contains("the embedder is down"), "{error}"); + assert!(matches!(whoami_index(&mut daemon), IndexStatus::Failing { .. })); + assert!(index.holding(&collection()).is_empty()); + + embedder.state.lock().unwrap().down = false; + daemon.index().wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 4, "every admitted document was kept and written"); +} + +/// A collection that is not this mind's is reported as refused, never +/// recreated, and re-read on the backoff: once it is gone the projection +/// completes. +#[test] +fn a_refused_collection_is_reported_and_reread() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + index.plant(&collection(), Described::Unlabelled); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries, quick()); + let mut daemon = Daemon::new(mind, sink); + daemon.index().wait_status(|status| matches!(status, IndexStatus::Refused { attempts, .. } if *attempts >= 2)); + assert!(matches!(whoami_index(&mut daemon), IndexStatus::Refused { pending: 4, .. })); + assert!(index.state.lock().unwrap().recreated.is_empty()); + + index.state.lock().unwrap().collections.remove(&collection()); + daemon.index().wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 4); +} + +// The adapters, against a stub that answers the routes it is given. + +struct Stub { + url: String, + seen: Arc>>, +} + +type Route = (&'static str, u16, &'static str); + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|window| window == needle) +} + +/// Serves each route once, in order, then repeats the last route that matched. +/// A route matches `" "` by prefix. +fn stub(routes: &[Route]) -> Stub { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let seen = Arc::new(Mutex::new(Vec::new())); + let routes: Arc>> = Arc::new(Mutex::new(routes.iter().map(|route| (*route, false)).collect())); + let log = Arc::clone(&seen); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { break }; + let (routes, log) = (Arc::clone(&routes), Arc::clone(&log)); + std::thread::spawn(move || answer_connection(stream, &routes, &log)); + } + }); + Stub { url, seen } +} + +fn answer_connection( + mut stream: TcpStream, + routes: &Mutex>, + log: &Mutex>, +) { + let mut buffer: Vec = Vec::new(); + loop { + let header_end = loop { + if let Some(at) = find(&buffer, b"\r\n\r\n") { + break at + 4; + } + let mut chunk = [0_u8; 4096]; + match stream.read(&mut chunk) { + Ok(0) | Err(_) => return, + Ok(read) => buffer.extend_from_slice(&chunk[..read]), + } + }; + let head = String::from_utf8_lossy(&buffer[..header_end]).into_owned(); + let length = head + .lines() + .find_map(|line| line.to_ascii_lowercase().strip_prefix("content-length:").map(|value| value.trim().parse::().unwrap())) + .unwrap_or(0); + while buffer.len() < header_end + length { + let mut chunk = [0_u8; 4096]; + match stream.read(&mut chunk) { + Ok(0) | Err(_) => return, + Ok(read) => buffer.extend_from_slice(&chunk[..read]), + } + } + let body = String::from_utf8_lossy(&buffer[header_end..header_end + length]).into_owned(); + buffer.drain(..header_end + length); + let mut first = head.lines().next().unwrap().split(' '); + let (method, path) = (first.next().unwrap().to_owned(), first.next().unwrap().to_owned()); + log.lock().unwrap().push((method.clone(), path.clone(), body)); + let key = format!("{method} {path}"); + let (status, answer) = { + let mut routes = routes.lock().unwrap(); + let matching: Vec = + routes.iter().enumerate().filter(|(_, ((prefix, ..), _))| key.starts_with(prefix)).map(|(at, _)| at).collect(); + let chosen = matching.iter().copied().find(|at| !routes[*at].1).or(matching.last().copied()); + match chosen { + Some(at) => { + routes[at].1 = true; + (routes[at].0.1, routes[at].0.2) + } + None => (404, "{}"), + } + }; + let response = + format!("HTTP/1.1 {status} X\r\ncontent-type: application/json\r\ncontent-length: {}\r\n\r\n{answer}", answer.len()); + if stream.write_all(response.as_bytes()).is_err() { + return; + } + } +} + +const TAGS: &str = r#"{"models":[{"name":"other:1","model":"other:1","digest":"zzz"},{"name":"m:1","model":"m:1","digest":"ac6d"}]}"#; +const SHOW: &str = r#"{"model_info":{"general.architecture":"qwen3","qwen3.embedding_length":4}}"#; +const EMBED: &str = r#"{"embeddings":[[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0],[0.5,1.0,2.0,3.0]]}"#; + +#[test] +fn ollama_identity_is_the_listed_digest_and_the_models_embedding_length() { + let ollama = stub(&[("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]); + let mut embedder = OllamaEmbedder::new(&ollama.url, "m:1"); + assert_eq!(embedder.model_identity(TIME).unwrap(), ModelIdentity { name: "m:1".into(), digest: "ac6d".into(), dimensions: 4 }); + let vectors = embedder.embed(&["a".into(), "b".into(), "c".into(), "d".into()], Duration::from_secs(5)).unwrap(); + assert_eq!(vectors.len(), 4); + assert_eq!(vectors[0], vec![0.5, 1.0, 2.0, 3.0]); + let seen = ollama.seen.lock().unwrap(); + let (_, _, body) = seen.iter().find(|(_, path, _)| path == "/api/embed").unwrap(); + assert!(body.contains("\"model\":\"m:1\"") && body.contains("\"input\":[\"a\",\"b\",\"c\",\"d\"]"), "{body}"); + drop(seen); + + // A listing entry that carries only `name` is still the model. + let bare = stub(&[ + ("GET /api/tags", 200, r#"{"models":[{"name":"m:1","digest":"ac6d"}]}"#), + ("POST /api/show", 200, SHOW), + ]); + assert_eq!(OllamaEmbedder::new(&bare.url, "m:1").model_identity(TIME).unwrap().digest, "ac6d"); + + let mut absent = OllamaEmbedder::new(&ollama.url, "not-pulled:1"); + assert!(format!("{:#}", absent.model_identity(TIME).unwrap_err()).contains("does not list the model")); + let refusing = stub(&[("POST /api/embed", 500, r#"{"error":"boom"}"#)]); + assert!(OllamaEmbedder::new(&refusing.url, "m:1").embed(&["a".into()], Duration::from_secs(5)).is_err()); +} + +/// Compatibility never reads the address: the same model at another address +/// leaves the collection alone, and the same address serving another digest +/// rebuilds it. +#[test] +fn the_embedders_address_is_not_part_of_the_collections_compatibility() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let routes = [("GET /api/tags", 200, TAGS), ("POST /api/show", 200, SHOW), ("POST /api/embed", 200, EMBED)]; + let repulled = TAGS.replace("ac6d", "beef"); + let index = FakeIndex::default(); + + let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); + first.want(entries.iter().cloned()); + drain(&mut first); + assert_eq!(counts(&index), (1, 1)); + + // Another address, the same model and digest. + let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); + moved.want(entries.iter().cloned()); + drain(&mut moved); + assert_eq!(counts(&index), (1, 1)); + + // The same model name re-pulled: another digest. + let leaked: &'static str = Box::leak(repulled.into_boxed_str()); + let other = [("GET /api/tags", 200, leaked), routes[1], routes[2]]; + let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); + rebuilt.want(entries.iter().cloned()); + drain(&mut rebuilt); + assert_eq!(counts(&index), (2, 2)); +} + +#[test] +fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { + let name = collection(); + let absent = stub(&[("GET /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); + assert_eq!(QdrantIndex::new(&absent.url).describe(&name, TIME).unwrap(), Described::Absent); + + let unlabelled = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":null}}}"#)]); + assert_eq!(QdrantIndex::new(&unlabelled.url).describe(&name, TIME).unwrap(), Described::Unlabelled); + let voidbot = stub(&[("GET /collections/", 200, r#"{"result":{"config":{"metadata":{"managedBy":"voidbot"}}}}"#)]); + assert_eq!(QdrantIndex::new(&voidbot.url).describe(&name, TIME).unwrap(), Described::Unlabelled); + + let labelled = stub(&[( + "GET /collections/", + 200, + r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","mind":"mind-commit-first","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, + )]); + assert_eq!( + QdrantIndex::new(&labelled.url).describe(&name, TIME).unwrap(), + Described::Labelled(CollectionMeta { + managed_by: "huginn".into(), + instance: "yggdrasil".into(), + mind: "mind-commit-first".into(), + model: "m:1".into(), + model_digest: "ac6d".into(), + dimensions: 4, + index_text_version: 1, + }) + ); + + // A scroll is followed to its last page. + let paged = stub(&[ + ("POST /collections/", 200, r#"{"result":{"points":[{"id":"a"},{"id":"b"}],"next_page_offset":"b"}}"#), + ("POST /collections/", 200, r#"{"result":{"points":[{"id":"c"}],"next_page_offset":null}}"#), + ]); + let ids = QdrantIndex::new(&paged.url).ids(&name).unwrap(); + let expected: BTreeSet = ["a", "b", "c"].map(String::from).into_iter().collect(); + assert_eq!(ids, expected); + let seen = paged.seen.lock().unwrap(); + assert!(!seen[0].2.contains("offset") && seen[1].2.contains("\"offset\":\"b\""), "{seen:?}"); + drop(seen); + + // A rebuild drops, then creates with the metadata and the vector shape. + let writing = stub(&[("DELETE /collections/", 200, "{}"), ("PUT /collections/", 200, "{}")]); + let meta = CollectionMeta { + managed_by: MANAGED_BY.into(), + instance: INSTANCE.into(), + mind: MIND.into(), + model: "m:1".into(), + model_digest: "ac6d".into(), + dimensions: 4, + index_text_version: INDEX_TEXT_VERSION, + }; + let mut store = QdrantIndex::new(&writing.url); + store.recreate(&name, &meta).unwrap(); + let point = Point { + id: point_id("d"), + vector: vec![1.0, 2.0, 3.0, 4.0], + payload: PointPayload { doc_id: "d".into(), kind: "question".into(), root: "eureka-state".into(), ordinal: 7, text_sha256: "h".into() }, + }; + store.upsert(&name, &[point]).unwrap(); + let seen = writing.seen.lock().unwrap(); + let paths: Vec = seen.iter().map(|(method, path, _)| format!("{method} {path}")).collect(); + assert_eq!( + paths, + [ + format!("DELETE /collections/{name}?wait=true"), + format!("PUT /collections/{name}?wait=true"), + format!("PUT /collections/{name}/points?wait=true"), + ] + ); + for expected in ["\"size\":4", "\"distance\":\"Cosine\"", "\"on_disk\":true", "\"managed_by\":\"huginn\"", "\"index_text_version\":1"] { + assert!(seen[1].2.contains(expected), "{expected} in {}", seen[1].2); + } + for expected in ["\"doc_id\":\"d\"", "\"kind\":\"question\"", "\"root\":\"eureka-state\"", "\"ordinal\":7", "\"text_sha256\":\"h\""] { + assert!(seen[2].2.contains(expected), "{expected} in {}", seen[2].2); + } + + let failing = stub(&[("PUT /collections/", 500, r#"{"status":{"error":"disk full"}}"#)]); + assert!(QdrantIndex::new(&failing.url).upsert(&name, &[]).is_err()); +} + +/// The collection's name is the contract with the store: one per instance, +/// `huginn_mind_`. +#[test] +fn the_collection_is_named_for_the_instance() { + assert_eq!(collection_name(&slug("yggdrasil")), "huginn_mind_yggdrasil"); + assert_eq!(collection_name(&slug("a.b")), "huginn_mind_a.b"); +} + +#[test] +fn a_step_error_says_what_stopped_it() { + assert_eq!(StepError::Refused("not ours".into()).to_string(), "not ours"); + let unavailable = StepError::Unavailable(anyhow::anyhow!("inner").context("outer")); + assert_eq!(unavailable.to_string(), "outer: inner"); +} + +/// What the sink reports is the worker's health with the inbox added to the +/// pending count, and a current index with something queued is behind. +#[test] +fn queued_entries_count_toward_every_status_that_has_a_pending_count() { + use IndexStatus as S; + assert_eq!(including(&S::Current, 0), S::Current); + assert_eq!(including(&S::Current, 3), S::Behind { pending: 3 }); + assert_eq!(including(&S::Reconciling { pending: 2 }, 3), S::Reconciling { pending: 5 }); + assert_eq!(including(&S::Behind { pending: 2 }, 3), S::Behind { pending: 5 }); + assert_eq!( + including(&S::Failing { pending: 2, attempts: 4, error: "e".into() }, 3), + S::Failing { pending: 5, attempts: 4, error: "e".into() } + ); + assert_eq!( + including(&S::Refused { pending: 2, attempts: 4, reason: "r".into() }, 3), + S::Refused { pending: 5, attempts: 4, reason: "r".into() } + ); +} + +/// Dropping the sink stops the worker: the worker's hold on the shared state +/// is released. A worker that never stopped would hold it forever. +#[test] +fn dropping_the_sink_stops_the_worker() { + let (embedder, index) = pair(); + let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let shared = Arc::downgrade(&sink.shared); + drop(sink); + eventually("the worker letting go of the shared state", || shared.upgrade().is_none()); +} + +/// F1. The promised curve, pure: 30 s doubling to a 10 min cap, never below +/// the first delay and never past the cap. A retry that shrank would hot-loop +/// against a dead Ollama. +#[test] +fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() { + let backoff = Backoff::default(); + assert_eq!((backoff.initial, backoff.max), (Duration::from_secs(30), Duration::from_secs(600))); + assert_eq!(backoff.recheck, Duration::from_secs(60), "an idle worker verifies the model and the collection each minute"); + let mut delay = backoff.initial; + let mut seen = vec![delay.as_secs()]; + for _ in 0..7 { + let next = backoff.after(delay); + assert!(next >= delay && next >= backoff.initial, "the delay never shrinks: {delay:?} -> {next:?}"); + assert!(next <= backoff.max, "the delay never passes the cap: {next:?}"); + delay = next; + seen.push(delay.as_secs()); + } + assert_eq!(seen, [30, 60, 120, 240, 480, 600, 600, 600]); + let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3), recheck: Duration::from_secs(3600), search_deadline: Duration::from_secs(30) }; + assert_eq!(small.after(Duration::from_millis(1)), Duration::from_millis(2)); + assert_eq!(small.after(Duration::from_millis(2)), Duration::from_millis(3)); +} + +/// F2. A collection lost while the daemon runs is found by the next failed +/// write, recreated and refilled from what the projector knows, without a +/// restart. +#[test] +fn a_collection_lost_after_startup_is_recreated_and_refilled() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 3); + + index.state.lock().unwrap().collections.remove(&collection()); + projector.want(entries[3..].iter().cloned()); + let error = projector.advance().expect_err("the write meets no collection"); + assert!(matches!(error, StepError::Unavailable(_)), "{error}"); + assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Failing { pending: 1, .. })); + + drain(&mut projector); + assert_eq!(index.state.lock().unwrap().recreated.len(), 2, "the collection was made again"); + assert_eq!(index.holding(&collection()), point_ids(&entries), "and holds every document, not only the new one"); + assert_eq!(projector.progress_status(), IndexStatus::Current); +} + +/// F2. A model re-pulled with other dimensions is seen before anything is +/// written: no vector of the new length is offered to the old collection, and +/// the next step rebuilds the collection to match. +#[test] +fn a_model_that_changes_dimensions_after_startup_rebuilds_the_collection() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + + embedder.state.lock().unwrap().identity.dimensions = 8; + projector.want(entries[3..].iter().cloned()); + let writes = index.state.lock().unwrap().upserts.len(); + assert_eq!(projector.advance().unwrap(), Advance::Progressed, "the change is seen, not written through"); + assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "no vector of 8 was offered to a collection of 4"); + drain(&mut projector); + + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2); + assert_eq!(state.recreated.last().unwrap().1.dimensions, 8); + let stored = &state.collections[&collection()]; + assert_eq!(stored.points.len(), 4); + assert!(stored.points.values().all(|point| point.vector.len() == 8)); +} + +/// F3. Two minds that share an instance name never share a collection: the +/// second is refused, and the first's points, label and history are untouched. +#[test] +fn a_second_mind_of_the_same_name_is_refused_and_leaves_the_first_alone() { + let (_first_root, first) = mind_with_documents(); + let (_second_root, second) = other_mind_of_the_same_name(); + let (first_id, second_id) = (first.genesis_receipt_id().unwrap().unwrap(), second.genesis_receipt_id().unwrap().unwrap()); + assert_ne!(first_id, second_id); + + let (embedder, index) = pair(); + let mut ours = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(first_id.clone())); + ours.want(first.index_entries(None).unwrap()); + drain(&mut ours); + let held = index.holding(&collection()); + assert_eq!(held.len(), 4); + + let intruder = FakeEmbedder::new("d1"); + let mut theirs = Projector::new(intruder.clone(), index.clone(), &slug(INSTANCE), Some(second_id)); + theirs.want(second.index_entries(None).unwrap()); + let error = theirs.advance().expect_err("another mind's collection is refused"); + assert!(matches!(error, StepError::Refused(_)), "{error}"); + assert!(matches!(theirs.failure_status(&error, 1), IndexStatus::Refused { .. })); + + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 1, "never rebuilt by the second mind"); + assert_eq!(state.upserts.len(), 1, "never written by the second mind"); + assert!(intruder.texts().is_empty()); + let Described::Labelled(label) = &state.collections[&collection()].label else { panic!() }; + assert_eq!(label.mind, first_id); + drop(state); + assert_eq!(index.holding(&collection()), held); +} + +/// A mind with no receipt has no identity: nothing is reconciled and no +/// collection is made, until the first admission reveals one. +#[test] +fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + assert_eq!(mind.genesis_receipt_id().unwrap(), None); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert_eq!(projector.advance().unwrap(), Advance::Idle); + let identified = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + assert_eq!(identified.progress_status(), IndexStatus::Behind { pending: 0 }, "an identified mind has yet to build its first index: lag, not a rebuild"); + assert!(index.state.lock().unwrap().collections.is_empty()); + + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); + let mut daemon = Daemon::new(mind, sink); + let seed = campaign_seed(); + let without_text = vec![seed[0].clone(), seed[1].clone()]; + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, without_text)), now()).answered()); + for _ in 0..500 { + if index.state.lock().unwrap().collections.contains_key(&collection()) { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let state = index.state.lock().unwrap(); + let Described::Labelled(label) = &state.collections[&collection()].label else { panic!("no collection was made") }; + assert_eq!(Some(&label.mind), daemon.mind().genesis_receipt_id().unwrap().as_ref()); +} + +/// Waits for a condition the worker will make true, without a clock in the +/// assertion: it polls for at most ten seconds and then says what never +/// happened. +fn eventually(what: &str, mut condition: impl FnMut() -> bool) { + for _ in 0..1000 { + if condition() { + return; + } + std::thread::sleep(Duration::from_millis(5)); + } + panic!("{what} never happened"); +} + +/// A model re-pulled under the same name and dimensions but a new digest must +/// not mix its vectors with the old model's. The check is before the write, +/// not after a failure: nothing is offered to the old collection, the +/// collection is rebuilt under the new digest, and every document is embedded +/// again. +#[test] +fn a_model_re_pulled_with_the_same_dimensions_never_writes_beside_the_old_vectors() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + + embedder.state.lock().unwrap().identity.digest = "d2".into(); + projector.want(entries[3..].iter().cloned()); + let writes = index.state.lock().unwrap().upserts.len(); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + assert_eq!(index.state.lock().unwrap().upserts.len(), writes, "the new vector was not written into the old collection"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 1 }, "the reconciliation was forgotten, not the entry"); + + drain(&mut projector); + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2); + assert_eq!(state.recreated.last().unwrap().1.model_digest, "d2"); + let stored = &state.collections[&collection()]; + assert_eq!(stored.points.len(), 4, "every document is in the rebuilt collection"); + assert_eq!(embedder.texts().len(), 3 + 4, "and was embedded again under the new digest"); +} + +/// Nothing is written while idle, so no write can fail to announce that the +/// collection is gone or the model has changed. The idle worker looks again on +/// its own clock: a lost collection is refilled and a re-pulled model rebuilds +/// it, with no admission to prompt either. +#[test] +fn an_idle_worker_finds_a_lost_collection_and_a_changed_model_without_a_write() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let recheck = Backoff { recheck: Duration::from_millis(20), ..quick() }; + let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries.clone(), recheck); + sink.wait_status(|status| *status == IndexStatus::Current); + assert_eq!(index.holding(&collection()), point_ids(&entries)); + + index.state.lock().unwrap().collections.remove(&collection()); + eventually("the lost collection being made again and refilled", || { + index.state.lock().unwrap().recreated.len() == 2 && index.holding(&collection()) == point_ids(&entries) + }); + + embedder.state.lock().unwrap().identity.digest = "d2".into(); + eventually("the collection being rebuilt under the new digest", || { + let state = index.state.lock().unwrap(); + state.recreated.len() == 3 + && state.recreated.last().is_some_and(|(_, meta)| meta.model_digest == "d2") + && state.collections[&collection()].points.len() == 4 + }); + sink.wait_status(|status| *status == IndexStatus::Current); +} + +/// A search embeds the query behind the model's instruction, asks the index +/// with no filter for `min(top_k * 4, 200)`, and hands back candidates by id +/// and kind; a hit of a kind this organ has no name for is left out. +#[test] +fn a_search_embeds_the_query_with_its_instruction_and_oversamples() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + index.state.lock().unwrap().hits = vec![ + Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }, + Hit { doc_id: "eureka-state:mystery:X".into(), kind: "mystery".into(), score: 0.8 }, + ]; + + let hits = projector.search("who owns the state", 3, SEARCH_DEADLINE).unwrap(); + assert_eq!(hits.len(), 1, "the unknown kind is left out"); + assert_eq!(hits[0].0.id.0, "eureka-state:question:Q1"); + assert_eq!(hits[0].0.kind, PipelineKind::Question); + assert_eq!(hits[0].1, 0.9); + let embedded = embedder.texts(); + assert_eq!(embedded.last().unwrap(), &format!("Instruct: {QUERY_INSTRUCTION}\nQuery: who owns the state")); + projector.search("again", 100, SEARCH_DEADLINE).unwrap(); + let searches = index.state.lock().unwrap().searches.clone(); + assert_eq!(searches.iter().map(|(name, length, limit)| (name.as_str(), *length, *limit)).collect::>(), [ + (collection().as_str(), 4, 12), + (collection().as_str(), 4, 200), + ]); +} + +/// A search is never run against a collection that is not known to fit the +/// model: before reconciliation, and after the model has changed, it refuses +/// by saying so rather than comparing vectors of two models. +#[test] +fn a_search_refuses_when_the_index_is_not_reconciled_with_the_model() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut fresh = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + assert!(format!("{:#}", fresh.search("anything", 3, SEARCH_DEADLINE).unwrap_err()).contains("not been built")); + assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); + + let mut projector = projected(&embedder, &index, &entries); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + let before = embedder.texts().len(); + assert!(format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()).contains("model changed")); + assert_eq!(embedder.texts().len(), before, "the query was not embedded under the new model"); + assert!(index.state.lock().unwrap().searches.is_empty()); + assert!(projector.stale(), "and the reconciliation is to be redone"); +} + +/// The search asks for the nearest points to a vector with no filter of any +/// kind (which documents may be shown is the mind's), and reads back the +/// document id, the kind and the score, in the order Qdrant gave them. A hit +/// missing what the join needs is an error, not a guess. +#[test] +fn qdrant_search_sends_no_filter_and_reads_ids_kinds_and_scores() { + let name = collection(); + let answering = stub(&[( + "POST /collections/", + 200, + r#"{"result":[{"id":"p1","score":0.75,"payload":{"doc_id":"eureka-state:question:Q1","kind":"question"}},{"id":"p2","score":0.5,"payload":{"doc_id":"eureka-state:ruling:R1","kind":"ruling"}}]}"#, + )]); + let hits = QdrantIndex::new(&answering.url).search(&name, &[0.5, 1.0, 2.0, 3.0], 12, TIME).unwrap(); + assert_eq!( + hits, + [ + Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.75 }, + Hit { doc_id: "eureka-state:ruling:R1".into(), kind: "ruling".into(), score: 0.5 }, + ] + ); + let seen = answering.seen.lock().unwrap(); + assert_eq!((seen[0].0.as_str(), seen[0].1.as_str()), ("POST", format!("/collections/{name}/points/search").as_str())); + for expected in ["\"vector\":[0.5,1.0,2.0,3.0]", "\"limit\":12", "\"with_vector\":false"] { + assert!(seen[0].2.contains(expected), "{expected} in {}", seen[0].2); + } + assert!(!seen[0].2.contains("filter"), "no predicate is lowered into the index: {}", seen[0].2); + drop(seen); + + let nameless = stub(&[("POST /collections/", 200, r#"{"result":[{"id":"p","score":0.5,"payload":{"kind":"question"}}]}"#)]); + assert!(QdrantIndex::new(&nameless.url).search(&name, &[1.0], 1, TIME).is_err()); + let failing = stub(&[("POST /collections/", 404, r#"{"status":{"error":"not found"}}"#)]); + assert!(QdrantIndex::new(&failing.url).search(&name, &[1.0], 1, TIME).is_err()); +} + +fn queued(ticket: u64, text: &str) -> Queued { + Queued { ticket: SearchTicket(ticket), text: text.into(), top_k: 3, due: Instant::now() + SEARCH_DEADLINE } +} + +fn shared_with( + inbox: Vec, + mind: Option, + searches: Vec, + closed: bool, +) -> Lock { + Arc::new(( + Mutex::new(Shared { + inbox, + mind, + status: IndexStatus::Current, + closed, + searches: searches.into(), + found: Vec::new(), + next_ticket: 0, + }), + Condvar::new(), + )) +} + +/// The idle wait is for work: an entry, the mind's identity, a search or the +/// sink closing ends it at once, and only a quiet worker waits out its +/// deadline. The deadline is the caller's: a deadline already past ends the +/// wait at once too, and waking for work does not extend it. +#[test] +fn an_idle_wait_ends_for_work_and_otherwise_at_its_deadline() { + let (_root, mind) = mind_with_documents(); + let entry = mind.index_entries(None).unwrap().remove(0); + let waits = |shared: &Lock| { + let started = Instant::now(); + wait_for_work(shared, Wait::Work { until: Instant::now() + Duration::from_secs(30) }); + started.elapsed() < Duration::from_secs(10) + }; + assert!(waits(&shared_with(vec![entry], None, Vec::new(), false)), "an entry"); + assert!(waits(&shared_with(Vec::new(), Some(MIND.into()), Vec::new(), false)), "the mind's identity"); + assert!(waits(&shared_with(Vec::new(), None, vec![queued(0, "q")], false)), "a search"); + assert!(waits(&shared_with(Vec::new(), None, Vec::new(), true)), "the sink closing"); + + let quiet = shared_with(Vec::new(), None, Vec::new(), false); + let started = Instant::now(); + wait_for_work(&quiet, Wait::Work { until: started + Duration::from_millis(60) }); + assert!(started.elapsed() >= Duration::from_millis(60), "a quiet worker waits for its deadline"); + let started = Instant::now(); + wait_for_work(&quiet, Wait::Work { until: started.checked_sub(Duration::from_secs(1)).unwrap_or(started) }); + assert!(started.elapsed() < Duration::from_secs(10), "a deadline already past is not waited for"); +} + +/// A backoff is the whole delay, whatever arrives: a failing index is retried +/// on its schedule. +#[test] +fn a_backoff_wait_is_the_whole_delay_whatever_arrives() { + let (_root, mind) = mind_with_documents(); + let entry = mind.index_entries(None).unwrap().remove(0); + let busy = shared_with(vec![entry], Some(MIND.into()), vec![queued(0, "q")], false); + let started = Instant::now(); + wait_for_work(&busy, Wait::Backoff(Duration::from_millis(80))); + assert!(started.elapsed() >= Duration::from_millis(80), "the delay was waited out although work was queued"); + let closed = shared_with(Vec::new(), None, Vec::new(), true); + let started = Instant::now(); + wait_for_work(&closed, Wait::Backoff(Duration::from_secs(30))); + assert!(started.elapsed() < Duration::from_secs(10), "a closed sink ends even a backoff"); +} + +/// Searches accepted while the index was healthy are answered, with the +/// reason, when the worker fails: nothing is left to wait for its deadline. +#[test] +fn queued_searches_are_answered_with_why_when_the_worker_gives_up() { + let shared = shared_with(Vec::new(), None, vec![queued(3, "a"), queued(4, "b")], false); + refuse_searches(&shared, "the index could not advance: down"); + let guard = lock(&shared); + assert!(guard.searches.is_empty()); + assert_eq!( + guard.found, + [ + (SearchTicket(3), Err("the index could not advance: down".to_string())), + (SearchTicket(4), Err("the index could not advance: down".to_string())), + ] + ); +} + +// The semantic read's index side: when a search is answered, how many wait, +// and what the worker does between them. + +type Answers = Vec<(SearchTicket, Result)>; + +/// The sink's own calls, named: the trait is generic over the store and the +/// worker does not care which. +fn ask_sink(sink: &mut WorkerSink, text: &str, top_k: u32) -> Result { + >::search(sink, text, top_k) +} + +fn collected(sink: &mut WorkerSink) -> Answers { + >::searched(sink) +} + +fn abandon(sink: &mut WorkerSink, ticket: SearchTicket) { + >::abandon(sink, ticket); +} + +/// Waits for `count` answers and returns them, in the order they were given. +fn answers(sink: &mut WorkerSink, count: usize) -> Answers { + let mut all = Vec::new(); + eventually("the searches being answered", || { + all.extend(collected(sink)); + all.len() >= count + }); + all +} + +/// `count` documents that share one document's text and differ in id, for the +/// tests whose subject is how many there are. +fn entries_of(count: usize) -> Vec { + let (_root, mind) = mind_with_documents(); + let template = mind.index_entries(None).unwrap().remove(0); + (0..count) + .map(|n| { + let mut entry = template.clone(); + entry.id.id.0 = format!("{}#{n}", entry.id.id.0); + entry + }) + .collect() +} + +fn queries(embedder: &FakeEmbedder) -> Vec { + embedder.texts().into_iter().filter(|text| text.starts_with("Instruct: ")).collect() +} + +/// A search is refused only while the index is being rebuilt or cannot be +/// asked, and the words name the state and how many documents are pending. Lag +/// is not a rebuild: the sink and the projector ask the one predicate and both +/// let it through. +#[test] +fn a_search_is_refused_only_for_a_rebuild_or_a_failure() { + use IndexStatus as S; + assert_eq!(unsearchable(&S::Current), None); + assert_eq!(unsearchable(&S::Behind { pending: 5 }), None, "lag is waited out, not refused"); + let says = |status: S, wanted: &[&str]| { + let why = unsearchable(&status).unwrap_or_else(|| panic!("{status:?} answered")); + for word in wanted { + assert!(why.contains(word), "{status:?}: {word} in {why}"); + } + }; + says(S::Reconciling { pending: 7 }, &["being rebuilt", "reconciling", "pending: 7"]); + says(S::Failing { pending: 3, attempts: 2, error: "boom".into() }, &["failing", "boom", "pending: 3"]); + says(S::Refused { pending: 4, attempts: 2, reason: "not ours".into() }, &["refused", "not ours", "pending: 4"]); + + // The sink: what the worker last published, and the documents admitted + // since, which are lag whatever the published state says of a current index. + let sink_in = |status: S, inbox: usize| { + let shared = shared_with(entries_of(inbox), None, Vec::new(), false); + lock(&shared).status = status; + WorkerSink { shared, identified: true, deadline: SEARCH_DEADLINE } + }; + assert!(ask_sink(&mut sink_in(S::Current, 2), "anything", 3).is_ok(), "admitted a moment ago and not yet absorbed"); + assert!(ask_sink(&mut sink_in(S::Behind { pending: 4 }, 0), "anything", 3).is_ok(), "the worker is writing"); + let refusal = ask_sink(&mut sink_in(S::Reconciling { pending: 4 }, 2), "anything", 3).unwrap_err(); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 6"), "{refusal}"); + let refusal = ask_sink(&mut sink_in(S::Failing { pending: 1, attempts: 1, error: "boom".into() }, 0), "anything", 3).unwrap_err(); + assert!(refusal.contains("failing") && refusal.contains("boom"), "{refusal}"); + + // The projector: a rebuild refuses without touching the ports to find out. + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let (lost_embedder, lost_index) = pair(); + let mut rebuilding_now = rebuilding(&lost_embedder, &lost_index, &entries); + let searches = lost_index.state.lock().unwrap().searches.len(); + let refusal = format!("{:#}", rebuilding_now.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); + assert_eq!(lost_index.state.lock().unwrap().searches.len(), searches, "the rebuilt collection was not asked"); + assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); + + // Lag it lets through: catching up is the worker's, before the search is taken. + let mut lagging = projected(&embedder, &index, &entries[..3]); + lagging.want(entries[3..].iter().cloned()); + assert_eq!(lagging.progress_status(), IndexStatus::Behind { pending: 1 }); + assert!(lagging.search("anything", 3, SEARCH_DEADLINE).is_ok()); +} + +/// A search that fails for its own reasons (the vector store errs, the +/// embedder is down, an identity read times out) fails that search alone: the +/// reconciliation stands, the status stays `Current`, the collection is not +/// listed again, and the next search is asked and answered. +#[test] +fn a_failed_search_fails_that_search_alone() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + type Fault = Box; + let faults: [(&str, Fault); 3] = [ + ("the vector store fails the search", Box::new(|_, index, on| index.state.lock().unwrap().fail_search = on)), + ("the embedder is down", Box::new(|embedder, _, on| embedder.state.lock().unwrap().down = on)), + ("the identity read never answers", Box::new(|embedder, _, on| embedder.state.lock().unwrap().hang_identity = on)), + ]; + for (what, fault) in faults { + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + let listed = index.state.lock().unwrap().listed; + fault(&embedder, &index, true); + assert!(projector.search("anything", 3, Duration::from_millis(50)).is_err(), "{what}"); + assert_eq!(projector.progress_status(), IndexStatus::Current, "{what}"); + assert!(!projector.stale(), "{what}"); + fault(&embedder, &index, false); + assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_ok(), "{what}: the next search is answered"); + assert_eq!(index.state.lock().unwrap().listed, listed, "{what}: the collection was not listed again"); + } + + // In the worker the failed search is answered and nothing else moves. + let (embedder, index) = pair(); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let listed = index.state.lock().unwrap().listed; + index.state.lock().unwrap().fail_search = true; + let ticket = ask_sink(&mut sink, "anything", 3).unwrap(); + let failed = answers(&mut sink, 1); + assert_eq!(failed[0].0, ticket); + assert!(failed[0].1.is_err()); + index.state.lock().unwrap().fail_search = false; + let next = ask_sink(&mut sink, "anything", 3).expect("the index is still current, not being rebuilt"); + let answered = answers(&mut sink, 1); + assert_eq!((answered[0].0, answered[0].1.is_ok()), (next, true)); + assert_eq!(index.state.lock().unwrap().listed, listed, "the worker did not list the collection again"); +} + +/// A model the search finds changed is a rebuild, and its refusal names the +/// state and how many documents are pending, as every other refusal does. +#[test] +fn a_model_changed_refusal_names_the_state_and_the_pending_count() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + projector.want(entries[3..].iter().cloned()); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + let refusal = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); + assert!(refusal.contains("model changed") && refusal.contains("being rebuilt") && refusal.contains("pending: 1"), "{refusal}"); +} + +/// A search asks the store what the collection says about itself: a label for +/// another mind, no label, or no collection is refused before the query is +/// embedded, and the reconciliation is to be redone. +#[test] +fn a_search_re_reads_the_collections_label() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let cases: [(&str, Box); 3] = [ + ( + "another mind's label", + Box::new(|stored| { + let Described::Labelled(meta) = &mut stored.label else { panic!() }; + meta.mind = "mind-commit-another".into(); + }), + ), + ("no label", Box::new(|stored| stored.label = Described::Unlabelled)), + ("another text version", Box::new(|stored| { + let Described::Labelled(meta) = &mut stored.label else { panic!() }; + meta.index_text_version += 1; + })), + ]; + for (what, change) in cases { + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + change(index.state.lock().unwrap().collections.get_mut(&collection()).unwrap()); + let embedded = embedder.texts().len(); + let refusal = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); + assert!(refusal.contains("no longer the one this mind's index was built as"), "{what}: {refusal}"); + assert_eq!(embedder.texts().len(), embedded, "{what}: nothing was embedded"); + assert!(index.state.lock().unwrap().searches.is_empty(), "{what}"); + assert!(projector.stale(), "{what}"); + } + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + index.state.lock().unwrap().collections.clear(); + assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_err(), "no collection"); + assert!(projector.stale()); +} + +/// The model is read again after the query is embedded: a vector made while the +/// model changed is never compared with the collection. +#[test] +fn a_model_that_changes_while_the_query_is_embedded_is_refused() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let gate = Gate::shut(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let index = FakeIndex::default(); + gate.open(); + let mut projector = projected(&embedder, &index, &entries); + gate.shut_again(); + let searching = std::thread::spawn(move || { + let found = projector.search("anything", 3, SEARCH_DEADLINE); + (projector, found) + }); + gate.wait_arrived(1); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + gate.open(); + let (projector, found) = searching.join().unwrap(); + let refusal = format!("{:#}", found.unwrap_err()); + assert!(refusal.contains("while the texts were embedded"), "{refusal}"); + assert!(index.state.lock().unwrap().searches.is_empty(), "the vector was not used"); + assert!(projector.stale()); +} + +/// Each search has a ticket of its own, and its answer comes back once, as the +/// worker gave it: nothing is added to it and nothing is left to be +/// collected twice. +#[test] +fn every_search_has_its_own_ticket_and_its_answer_comes_back_exactly_once() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + index.state.lock().unwrap().hits = + vec![Hit { doc_id: "eureka-state:question:Q1".into(), kind: "question".into(), score: 0.9 }]; + let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), entries, quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let expected: Hits = vec![( + PipelineRef { kind: PipelineKind::Question, id: Short("eureka-state:question:Q1".into()) }, + 0.9, + )]; + let mut tickets = Vec::new(); + for text in ["a", "b", "c"] { + let ticket = ask_sink(&mut sink, text, 3).unwrap(); + assert_eq!(answers(&mut sink, 1), vec![(ticket, Ok(expected.clone()))]); + assert!(collected(&mut sink).is_empty(), "an answer is handed over once"); + tickets.push(ticket); + } + assert_eq!(tickets, [SearchTicket(0), SearchTicket(1), SearchTicket(2)]); +} + +/// The queue of searches is bounded and the asker who does not fit is told. +/// One search is inside the embedder and the queue is full; the next is +/// refused by name, and when the worker has answered some it takes more. +#[test] +fn the_search_queue_is_bounded_and_the_asker_is_told_when_it_is_full() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + ask_sink(&mut sink, "in flight", 3).unwrap(); + gate.wait_arrived(1); + for n in 0..SEARCHES_QUEUED_MAX { + ask_sink(&mut sink, &format!("waiting {n}"), 3).unwrap_or_else(|why| panic!("{n} of {SEARCHES_QUEUED_MAX}: {why}")); + } + let refusal = ask_sink(&mut sink, "one too many", 3).unwrap_err(); + assert!(refusal.contains(&format!("{SEARCHES_QUEUED_MAX} searches waiting")), "{refusal}"); + + gate.open(); + let answered = answers(&mut sink, SEARCHES_QUEUED_MAX + 1); + assert_eq!(answered.len(), SEARCHES_QUEUED_MAX + 1); + assert!(ask_sink(&mut sink, "room again", 3).is_ok()); +} + +/// A search the asker has stopped waiting for is taken out of the queue and +/// never embedded; the ones on either side of it are. +#[test] +fn an_abandoned_search_is_never_embedded() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let first = ask_sink(&mut sink, "first", 3).unwrap(); + gate.wait_arrived(1); + let second = ask_sink(&mut sink, "second", 3).unwrap(); + let third = ask_sink(&mut sink, "third", 3).unwrap(); + abandon(&mut sink, second); + gate.open(); + let answered = answers(&mut sink, 2); + let tickets: Vec = answered.iter().map(|(ticket, _)| *ticket).collect(); + assert_eq!(tickets, [first, third]); + let texts = queries(&embedder); + assert_eq!(texts.len(), 2, "{texts:?}"); + assert!(texts[0].ends_with("Query: first") && texts[1].ends_with("Query: third"), "{texts:?}"); +} + +/// The worker serves one queued search and returns to its writes: three +/// waiting searches leave two after one turn. +#[test] +fn one_queued_search_is_served_per_turn() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a"), queued(2, "b"), queued(3, "c")], false); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert_eq!(guard.searches.iter().map(|queued| queued.ticket).collect::>(), [SearchTicket(2), SearchTicket(3)]); + assert_eq!(guard.found.iter().map(|(ticket, _)| *ticket).collect::>(), [SearchTicket(1)]); +} + +/// A search reads its own writes. Asked while a batch is in the embedder and +/// eight more documents wait, it is accepted, waits behind them, and is +/// answered only once every one is in the collection. +#[test] +fn a_search_waits_for_the_writes_ahead_of_it_and_reads_them() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), quick()); + gate.wait_arrived(1); + let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); + gate.open(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, ticket); + assert!(answered[0].1.is_ok(), "{answered:?}"); + assert_eq!(index.state.lock().unwrap().held_when_searched, [BATCH + 8], "the search saw every document admitted before it"); + sink.wait_status(|status| *status == IndexStatus::Current); +} + +/// The worker takes admissions before the search, and leaves the search queued +/// while the writes ahead of it are unfinished: a search asked with a document +/// still in the inbox absorbs it and waits, and once the index has caught up +/// it is answered. An index being rebuilt is not waited on: the search is taken +/// and refused, naming the state and the pending count. +#[test] +fn the_worker_absorbs_admissions_before_a_search_and_waits_for_lag_but_not_for_a_rebuild() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + let shared = shared_with(vec![entries[3].clone()], None, vec![queued(1, "a")], false); + + serve_search(&mut projector, &shared); + { + let guard = lock(&shared); + assert!(guard.inbox.is_empty(), "the admission ahead of the search was absorbed"); + assert_eq!(guard.searches.len(), 1, "the search waits for it to be written"); + assert!(guard.found.is_empty()); + } + assert_eq!(projector.pending(), 1); + + drain(&mut projector); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(guard.searches.is_empty()); + assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); + assert_eq!(index.state.lock().unwrap().held_when_searched, [4]); + drop(guard); + + // A rebuild: the collection was lost and made again, and every entry is owed. + let (rebuild_embedder, rebuild_index) = pair(); + let mut rebuilding = rebuilding(&rebuild_embedder, &rebuild_index, &entries); + let shared = shared_with(Vec::new(), None, vec![queued(2, "b")], false); + serve_search(&mut rebuilding, &shared); + let guard = lock(&shared); + let [(SearchTicket(2), Err(refusal))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 4"), "{refusal}"); +} + +/// A collection made again with nothing to refill owes nothing, so the rebuild +/// ends inside `reconcile`: the next entry is lag, and a search waits for it +/// rather than being refused as a rebuild. +#[test] +fn a_rebuild_that_owes_nothing_settles_when_it_is_made() { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into())); + drain(&mut projector); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + assert_eq!(projector.recheck().unwrap(), Advance::Progressed, "the collection is made again under the new digest"); + assert_eq!(projector.pending(), 0); + + projector.want(entries_of(1)); + assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 1 }, "nothing was owed, so nothing is being rebuilt"); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert_eq!(guard.searches.len(), 1, "the search waits for the write"); + assert!(guard.found.is_empty(), "and is not refused: {:?}", guard.found); +} + +/// An identified mind with nothing owed has still to reconcile once: a search +/// waits for that, and is not taken to fail against an index not yet built. +#[test] +fn a_search_waits_for_the_first_reconcile_of_an_identified_mind_owing_nothing() { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); + assert_eq!(projector.pending(), 0); + assert!(projector.stale()); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + serve_search(&mut projector, &shared); + { + let guard = lock(&shared); + assert_eq!(guard.searches.len(), 1, "the search waits for the reconcile"); + assert!(guard.found.is_empty(), "{:?}", guard.found); + } + drain(&mut projector); + serve_search(&mut projector, &shared); + assert!(matches!(lock(&shared).found.as_slice(), [(SearchTicket(1), Ok(_))])); +} + +/// A search that was queued when the worker failed is answered with why, by +/// the worker's own failure path. +#[test] +fn a_search_queued_before_the_worker_fails_is_answered_with_why() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder.clone(), index, &slug(INSTANCE), Some(MIND.into()), entries_of(1), quick()); + gate.wait_arrived(1); + lock(&sink.shared).searches.push_back(queued(5, "anything")); + embedder.state.lock().unwrap().down = true; + gate.open(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, SearchTicket(5)); + let Err(why) = &answered[0].1 else { panic!("{answered:?}") }; + assert!(why.contains("the semantic index is failing") && why.contains("pending: 1") && why.contains("the embedder is down"), "{why}"); +} + +/// While a failure is being reported, the report's pending count is the one +/// the worker holds now, published under the lock that took the entries out of +/// the inbox: never the count from before they moved. +#[test] +fn a_failure_report_carries_the_pending_count_it_has_now() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); + projector.want(entries[..2].iter().cloned()); + let shared = shared_with(entries[2..].to_vec(), None, Vec::new(), false); + let error = StepError::Unavailable(anyhow::anyhow!("down")); + assert!(absorb(&mut projector, &shared, Some((&error, 3)))); + { + let guard = lock(&shared); + assert!(guard.inbox.is_empty()); + assert_eq!(guard.status, IndexStatus::Failing { pending: 4, attempts: 3, error: "down".into() }); + } + assert!(absorb(&mut projector, &shared, None)); + assert_eq!(lock(&shared).status, IndexStatus::Behind { pending: 4 }, "a first build is lag"); + lock(&shared).closed = true; + assert!(!absorb(&mut projector, &shared, None), "a closed sink ends the worker"); +} + +/// The worker's recheck is on its own clock: searches arriving faster than the +/// interval do not keep pushing it out. +#[test] +fn steady_searches_do_not_postpone_the_workers_recheck() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let backoff = Backoff { recheck: Duration::from_millis(200), ..quick() }; + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); + sink.wait_status(|status| *status == IndexStatus::Current); + let listed = index.state.lock().unwrap().listed; + let started = Instant::now(); + while index.state.lock().unwrap().listed < listed + 2 { + assert!(started.elapsed() < Duration::from_secs(5), "the worker never rechecked while searches kept arriving"); + let _ = ask_sink(&mut sink, "anything", 3); + collected(&mut sink); + std::thread::sleep(Duration::from_millis(10)); + } +} + +/// `wait_status` gives up at one deadline, not one per wake: a health that +/// keeps changing does not keep it waiting. +#[test] +fn wait_status_gives_up_at_its_overall_deadline_however_often_the_health_changes() { + let (embedder, index) = pair(); + let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); + sink.wait_status(|status| *status == IndexStatus::Current); + let shared = Arc::clone(&sink.shared); + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let ticking = Arc::clone(&stop); + let ticker = std::thread::spawn(move || { + while !ticking.load(std::sync::atomic::Ordering::Relaxed) { + shared.1.notify_all(); + std::thread::sleep(Duration::from_millis(5)); + } + }); + let (tell, told) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let gave_up = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + sink.wait_status_for(Duration::from_millis(300), |status| *status == IndexStatus::Reconciling { pending: 99 }) + })) + .is_err(); + let _ = tell.send(gave_up); + }); + let gave_up = told.recv_timeout(Duration::from_secs(10)); + stop.store(true, std::sync::atomic::Ordering::Relaxed); + ticker.join().unwrap(); + assert_eq!(gave_up, Ok(true), "the wait did not end at its deadline"); +} + +/// An idle worker verifies once per interval, not in a loop: the recheck moves +/// its own deadline when it runs. +#[test] +fn an_idle_worker_rechecks_once_per_interval() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let backoff = Backoff { recheck: Duration::from_millis(100), ..quick() }; + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries, backoff); + sink.wait_status(|status| *status == IndexStatus::Current); + std::thread::sleep(Duration::from_millis(600)); + let listed = index.state.lock().unwrap().listed; + assert!((2..=40).contains(&listed), "the startup reconciliation and about five rechecks listed the collection {listed} times"); +} + +// The search's rulings: what may be waiting ahead of it, when a rebuild +// refuses, what a first build is, how time is spent, and who owns the deadline. + +/// A search is never taken while a document sits in the inbox. An admission +/// that lands after the inbox was absorbed and before the search was taken (the +/// hook puts one there, where a second lock once left the gap) leaves the +/// search queued; the next turn absorbs it, and the search is answered only +/// once it is written. +#[test] +fn a_search_is_never_taken_while_an_admission_sits_in_the_inbox() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + let late = entries[3].clone(); + AFTER_ABSORB.with(|hook| *hook.borrow_mut() = Some(Box::new(move |guard| guard.inbox.push(late)))); + + serve_search(&mut projector, &shared); + { + let guard = lock(&shared); + assert_eq!(guard.searches.len(), 1, "the search was not taken past the admission"); + assert!(guard.found.is_empty()); + assert_eq!(guard.inbox.len(), 1); + } + serve_search(&mut projector, &shared); + assert_eq!(lock(&shared).searches.len(), 1, "absorbed, but not yet written"); + drain(&mut projector); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(matches!(guard.found.as_slice(), [(SearchTicket(1), Ok(_))]), "{:?}", guard.found); + assert_eq!(index.state.lock().unwrap().held_when_searched, [4], "the search saw the admission"); +} + +/// Through the sink, an admission acknowledged before the search was asked is +/// in the inbox while the worker is inside a batch; the search reads it. +#[test] +fn a_search_reads_an_admission_acknowledged_before_it_was_asked() { + let gate = Gate::shut(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(2), quick()); + gate.wait_arrived(1); + let admitted = entries_of(5).split_off(2); + lock(&sink.shared).inbox.extend(admitted); + let ticket = ask_sink(&mut sink, "anything", 3).expect("lag is not a rebuild"); + gate.open(); + let answered = answers(&mut sink, 1); + assert_eq!(answered[0].0, ticket); + assert!(answered[0].1.is_ok(), "{answered:?}"); + assert_eq!(index.state.lock().unwrap().held_when_searched, [5]); +} + +/// A collection made again is a rebuild until every document it owes is +/// written again: searches are refused as rebuilding, however the collection +/// came to be made again, and however many steps or failures the refill takes. +#[test] +fn a_collection_made_again_refuses_searches_until_it_is_refilled() { + let refused = |projector: &mut Projector, pending: u32| { + let why = format!("{:#}", projector.search("anything", 3, SEARCH_DEADLINE).unwrap_err()); + assert!(why.contains("being rebuilt") && why.contains(&format!("pending: {pending}")), "{why}"); + }; + let entries = entries_of(BATCH + 8); + + // Lost while the worker ran. + let (embedder, index) = pair(); + let mut projector = rebuilding(&embedder, &index, &entries); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); + refused(&mut projector, 40); + index.state.lock().unwrap().down = true; + projector.advance().expect_err("the store is down mid-refill"); + index.state.lock().unwrap().down = false; + assert_eq!(projector.advance().unwrap(), Advance::Progressed, "reconciled again, under a label that already fits"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }, "a failure does not end the rebuild"); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }, "half refilled is still a rebuild"); + refused(&mut projector, 8); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_ok(), "answered once refilled"); + assert_eq!(index.state.lock().unwrap().held_when_searched, [40]); + projector.want(entries_of(1).into_iter().map(|mut entry| { + entry.id.id.0.push_str("-later"); + entry + })); + assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 1 }, "a later admission is lag, not a rebuild"); + + // Made again because the model changed. + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + embedder.state.lock().unwrap().identity.digest = "d2".into(); + assert_eq!(projector.recheck().unwrap(), Advance::Progressed, "the change is seen and the collection made again"); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 40 }); + projector.advance().unwrap(); + assert_eq!(projector.progress_status(), IndexStatus::Reconciling { pending: 8 }); + refused(&mut projector, 8); + drain(&mut projector); + assert!(projector.search("anything", 3, SEARCH_DEADLINE).is_ok()); +} + +/// Through the worker: a collection lost while the worker runs is made again, +/// and a search asked during the refill is refused as rebuilding, naming how +/// many documents are pending; one asked after it is answered. +#[test] +fn a_search_asked_during_a_refill_is_refused_and_one_after_it_is_answered() { + let gate = Gate::shut(); + gate.open(); + let (_, index) = pair(); + let embedder = FakeEmbedder::new("d1").behind(&gate); + let backoff = Backoff { recheck: Duration::from_millis(50), ..quick() }; + let mut sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), Some(MIND.into()), entries_of(BATCH + 8), backoff); + sink.wait_status(|status| *status == IndexStatus::Current); + gate.shut_again(); + index.state.lock().unwrap().collections.remove(&collection()); + gate.wait_arrived(1); + let refusal = ask_sink(&mut sink, "anything", 3).unwrap_err(); + assert!(refusal.contains("being rebuilt") && refusal.contains("pending: 40"), "{refusal}"); + gate.open(); + sink.wait_status(|status| *status == IndexStatus::Current); + let ticket = ask_sink(&mut sink, "anything", 3).expect("refilled"); + let answered = answers(&mut sink, 1); + assert_eq!((answered[0].0, answered[0].1.is_ok()), (ticket, true)); + assert_eq!(index.state.lock().unwrap().held_when_searched.last(), Some(&40)); +} + +/// A mind whose index has never been built has nothing stale to protect: its +/// first build is lag, whether the search was asked before the worker had +/// looked at the first admission or after. Only a rebuild of an index that +/// exists refuses. +#[test] +fn a_minds_first_build_is_lag_whoever_looks_first() { + for asked_before_the_worker_looked in [true, false] { + let what = if asked_before_the_worker_looked { "asked first" } else { "asked after the worker looked" }; + let (embedder, index) = pair(); + let entries = entries_of(3); + let (mut projector, shared) = if asked_before_the_worker_looked { + let projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + (projector, shared_with(entries, Some(MIND.into()), Vec::new(), false)) + } else { + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + projector.want(entries); + let shared = shared_with(Vec::new(), None, Vec::new(), false); + lock(&shared).status = projector.progress_status(); + (projector, shared) + }; + let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline: SEARCH_DEADLINE }; + let ticket = ask_sink(&mut sink, "anything", 3).unwrap_or_else(|why| panic!("{what}: {why}")); + + serve_search(&mut projector, &shared); + assert_eq!(projector.progress_status(), IndexStatus::Behind { pending: 3 }, "{what}: not yet reconciled, and lag"); + assert_eq!(lock(&shared).searches.len(), 1, "{what}: the search waits for the first reconciliation"); + assert_eq!(projector.advance().unwrap(), Advance::Progressed); + serve_search(&mut projector, &shared); + assert_eq!(lock(&shared).searches.len(), 1, "{what}: and for the first writes"); + drain(&mut projector); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(matches!(guard.found.as_slice(), [(found, Ok(_))] if *found == ticket), "{what}: {:?}", guard.found); + assert_eq!(index.state.lock().unwrap().held_when_searched, [3], "{what}"); + } + + let entries = entries_of(4); + let (embedder, index) = pair(); + projected(&embedder, &index, &entries[..3]); + let mut partial = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); + partial.want(entries.iter().cloned()); + partial.advance().unwrap(); + assert_eq!(partial.progress_status(), IndexStatus::Behind { pending: 1 }, "an index found whole at start-up only lags"); + drain(&mut partial); + index.state.lock().unwrap().collections.remove(&collection()); + assert_eq!(partial.recheck().unwrap(), Advance::Progressed); + assert_eq!(partial.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index found at start-up was built: losing it is a rebuild"); + + embedder.state.lock().unwrap().identity.digest = "d2".into(); + let mut replaced = Projector::new(embedder, index, &slug(INSTANCE), Some(MIND.into())); + replaced.want(entries); + replaced.advance().unwrap(); + assert_eq!(replaced.progress_status(), IndexStatus::Reconciling { pending: 4 }, "an index that exists and is made again is a rebuild"); +} + +/// A mind with no identity and documents owed has nothing to wait for: the +/// search is taken and refused, not left queued for a worker that would spin +/// on it. +#[test] +fn a_search_is_refused_when_documents_are_owed_to_a_mind_with_no_identity() { + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + projector.want(entries_of(2)); + let shared = shared_with(Vec::new(), None, vec![queued(1, "a")], false); + serve_search(&mut projector, &shared); + let guard = lock(&shared); + assert!(guard.searches.is_empty()); + let [(SearchTicket(1), Err(why))] = guard.found.as_slice() else { panic!("{:?}", guard.found) }; + assert!(why.contains("being rebuilt") && why.contains("pending: 2"), "{why}"); + assert!(embedder.texts().is_empty() && index.state.lock().unwrap().searches.is_empty()); +} + +/// Every call a search makes is bounded by what its time has left, not by the +/// time it was given: after a slow identity read the embed's bound is smaller +/// by what the read took, and the query's by both reads. +#[test] +fn a_later_calls_bound_shrinks_as_the_search_spends_its_time() { + let entries = entries_of(3); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries); + let delay = Duration::from_millis(150); + let within = Duration::from_secs(10); + { + let mut state = embedder.state.lock().unwrap(); + state.identity_delay = delay; + state.bounds.clear(); + state.identity_bounds.clear(); + } + index.state.lock().unwrap().bounds.clear(); + projector.search("anything", 3, within).unwrap(); + + let embedder = embedder.state.lock().unwrap(); + let index = index.state.lock().unwrap(); + assert!(embedder.identity_bounds[0] <= within); + assert!(embedder.bounds[0] <= within - delay, "the embed came after one slow read: {:?}", embedder.bounds); + assert!(embedder.identity_bounds[1] <= within - delay, "{:?}", embedder.identity_bounds); + let queried = index.bounds.iter().find(|(call, _)| *call == "search").unwrap().1; + assert!(queried <= within - delay * 2, "the query came after two slow reads: {queried:?}"); +} + +/// The deadline has one owner: the sink. It stamps every search with it, and +/// it is what the serve loop is told (`Daemon::search_deadline`), so the loop's +/// wait and the worker's bound are one value. +#[test] +fn the_sink_owns_the_searchs_deadline_and_tells_the_loop() { + let (_root, mind) = mind_with_documents(); + let deadline = Duration::from_secs(7); + let shared = shared_with(Vec::new(), None, Vec::new(), false); + let mut sink = WorkerSink { shared: Arc::clone(&shared), identified: true, deadline }; + let asked = Instant::now(); + ask_sink(&mut sink, "anything", 3).unwrap(); + let due = lock(&shared).searches.back().expect("no worker took it").due; + assert!(due >= asked + deadline && due <= Instant::now() + deadline, "stamped with the sink's deadline"); + assert_eq!(>::search_deadline(&sink), deadline); + assert_eq!(Daemon::new(mind, sink).search_deadline(), deadline); +} From debe7ce9e7160c5c63832f88d67f9add6f97b1aa Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:30:21 +0200 Subject: [PATCH 025/520] Move the operation envelope to huginn-mind: one spelling for client and daemon envelope.rs (encode/decode request and response, OperationFailure, FAILURE_SCHEMA) leaves huginn-daemon for huginn-mind, next to the wire types it wraps. The daemon imports it and no longer re-exports it. base64 moves with it; the daemon keeps base64 because serve.rs measures and decodes payloads. schemars leaves the daemon, whose only user was the envelope's OperationFailure derive. The two envelope tests run unchanged from their new home on huginn-mind fixtures. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 +- crates/huginn-daemon/Cargo.toml | 1 - crates/huginn-daemon/src/lib.rs | 10 +++------- crates/huginn-daemon/src/semantic_tests.rs | 2 +- crates/huginn-daemon/src/serve.rs | 8 ++++---- crates/huginn-mind/Cargo.toml | 1 + .../src/envelope.rs | 16 ++++++++++------ crates/huginn-mind/src/lib.rs | 4 +++- 8 files changed, 23 insertions(+), 21 deletions(-) rename crates/{huginn-daemon => huginn-mind}/src/envelope.rs (93%) diff --git a/Cargo.lock b/Cargo.lock index 3ecdc74..1e30f52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -477,7 +477,6 @@ dependencies = [ "cultnet-rs", "huginn-mind", "rmp-serde", - "schemars", "serde", "serde_json", "sha2", @@ -491,6 +490,7 @@ name = "huginn-mind" version = "0.1.0" dependencies = [ "anyhow", + "base64", "chrono", "cultcache-rs", "cultnet-rs", diff --git a/crates/huginn-daemon/Cargo.toml b/crates/huginn-daemon/Cargo.toml index 428588e..5890c67 100644 --- a/crates/huginn-daemon/Cargo.toml +++ b/crates/huginn-daemon/Cargo.toml @@ -24,7 +24,6 @@ chrono = "0.4.44" cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "6251cbcd37c12ef3f0c1cc50063a03369225b7ac", package = "cultnet-rs" } huginn-mind = { path = "../huginn-mind" } rmp-serde = "1" -schemars = "1" serde = { version = "1", features = ["derive"] } # The vector store's and the embedder's HTTP APIs are the xenos boundary, and # JSON exists in `index/ollama.rs` and `index/qdrant.rs` only. `ureq` 3 with diff --git a/crates/huginn-daemon/src/lib.rs b/crates/huginn-daemon/src/lib.rs index a9fbb9f..90ae5e8 100644 --- a/crates/huginn-daemon/src/lib.rs +++ b/crates/huginn-daemon/src/lib.rs @@ -1,14 +1,12 @@ //! `huginn-daemon`: one process that serves one mind over CultNet. //! -//! The crate owns five things and no rule. `startup` owns "may this process +//! The crate owns four things and no rule. `startup` owns "may this process //! serve this mind", by delegating to `Mind::open` and refusing before //! anything listens. `index` owns when the mind's documents are embedded and //! written to the vector store, off the serving path. `Daemon::handle` owns "which `Mind` method answers this //! request": it dispatches, it asks `Mind::require_instance` for every read -//! that names an instance, and it compares nothing itself. `envelope` owns "is -//! this a request at all": a message it cannot decode is answered with a typed -//! failure and touches no mind. `serve` owns "which session gets which reply" -//! and whether an answer is delivered whole, deferred or refused; `bodies` +//! that names an instance, and it compares nothing itself. `serve` owns "which +//! session gets which reply" and whether an answer is delivered whole, deferred or refused; `bodies` //! owns how long and how much of a deferred answer is held. //! //! Every admission rule, the instance check, the receipt, the derived status @@ -17,7 +15,6 @@ pub mod bodies; pub mod daemon; -pub mod envelope; pub mod index; pub mod serve; @@ -25,5 +22,4 @@ pub mod serve; mod semantic_tests; pub use daemon::{Daemon, Handled, Hits, IndexSink, Search, SearchTicket, runtime_id}; -pub use envelope::{FAILURE_SCHEMA, OperationFailure}; pub use serve::{Options, ServeOptions, parse_options, run, startup}; diff --git a/crates/huginn-daemon/src/semantic_tests.rs b/crates/huginn-daemon/src/semantic_tests.rs index 513f0e9..f72c5b4 100644 --- a/crates/huginn-daemon/src/semantic_tests.rs +++ b/crates/huginn-daemon/src/semantic_tests.rs @@ -20,7 +20,7 @@ use tempfile::TempDir; use crate::daemon::Daemon; use crate::daemon::tests::{CAMPAIGN, INSTANCE, batch, campaign_seed, now, slug}; -use crate::envelope::{decode_response, encode_request}; +use huginn_mind::envelope::{decode_response, encode_request}; use crate::index::fakes::{FakeEmbedder, FakeIndex, Gate}; use crate::index::{Backoff, Described, Hit, WorkerSink, collection_name}; use crate::serve::{ServeOptions, bind, run, schema_registry}; diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index a5efc02..bfc54b8 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -79,7 +79,7 @@ use crate::bodies::{ DEFERRED_BUDGET_BYTES, DEFERRED_CHUNK_BYTES, DeferredBodies, MAX_DEFERRED_BODY_BYTES, }; use crate::daemon::{Daemon, Handled, IndexSink, Search, SearchTicket, runtime_id}; -use crate::envelope::{OperationFailure, decode_request, encode_failure, encode_response}; +use huginn_mind::envelope::{OperationFailure, decode_request, encode_failure, encode_response}; use crate::index::{Backoff, Embedder, VectorIndex, WorkerSink}; /// What the loop does when nothing is waiting. A deferred body untouched for @@ -345,7 +345,7 @@ fn encode_or_fail( Err(error) => encode_failure( message_id, operation, - &crate::envelope::OperationFailure { + &OperationFailure { code: "response-not-encodable".into(), message: format!("{error:#}"), }, @@ -548,7 +548,7 @@ mod tests { open_unindexed, seeded_wide, slug, }; use huginn_mind::epiphany_pipeline::{PipelineKind, PipelineRef}; - use crate::envelope::{FAILURE_SCHEMA, decode_response, encode_request}; + use huginn_mind::envelope::{FAILURE_SCHEMA, decode_response, encode_request}; use crate::index::Backoff; use base64::Engine; use base64::engine::general_purpose::STANDARD; @@ -632,7 +632,7 @@ mod tests { assert_eq!(rejected(&reply).1, code); } // The sixth code, `not-an-operation-request`, is the codec's and is - // pinned in `envelope::tests`: `answer` never produces it, because a + // pinned in `huginn_mind::envelope::tests`: `answer` never produces it, because a // message of another family is answered with `Error` instead. let reply = answer(&mut daemon, ®istry, &mut fresh_bodies(), CultNetMessage::Error { error: "hello".into(), code: None, details: None }, now()).reply(); let CultNetMessage::Error { error, .. } = &reply else { panic!("expected an error, got {reply:?}") }; diff --git a/crates/huginn-mind/Cargo.toml b/crates/huginn-mind/Cargo.toml index a2e60d4..ccf4e94 100644 --- a/crates/huginn-mind/Cargo.toml +++ b/crates/huginn-mind/Cargo.toml @@ -15,6 +15,7 @@ publish.workspace = true # daemon's (Cut 10, Cut 11). [dependencies] anyhow = "1" +base64 = "0.22" chrono = "0.4.44" cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "6251cbcd37c12ef3f0c1cc50063a03369225b7ac", package = "cultcache-rs" } cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "6251cbcd37c12ef3f0c1cc50063a03369225b7ac", package = "cultnet-rs" } diff --git a/crates/huginn-daemon/src/envelope.rs b/crates/huginn-mind/src/envelope.rs similarity index 93% rename from crates/huginn-daemon/src/envelope.rs rename to crates/huginn-mind/src/envelope.rs index fecbde2..8152a54 100644 --- a/crates/huginn-daemon/src/envelope.rs +++ b/crates/huginn-mind/src/envelope.rs @@ -14,7 +14,7 @@ use anyhow::Result; use base64::Engine; use base64::engine::general_purpose::STANDARD; use cultnet_rs::CultNetMessage; -use huginn_mind::wire::{HuginnMindRequest, HuginnMindResponse, MIND_REQUEST_SCHEMA, MIND_RESPONSE_SCHEMA, MIND_SERVICE_ID}; +use crate::wire::{HuginnMindRequest, HuginnMindResponse, MIND_REQUEST_SCHEMA, MIND_RESPONSE_SCHEMA, MIND_SERVICE_ID}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -151,9 +151,9 @@ pub fn decode_response(message: &CultNetMessage) -> Result<(String, Result MindRefusal { MindRefusal::ForeignInstance { declared: OTHER.into(), mind: INSTANCE.into() } @@ -183,7 +183,7 @@ mod tests { schema_epoch: "epiphany.pipeline.epoch.v2".into(), documents: 0, receipts: 0, - index: huginn_mind::wire::IndexStatus::Current, + index: crate::wire::IndexStatus::Current, }), "accepted", ), @@ -214,7 +214,11 @@ mod tests { /// `Error` instead. #[test] fn a_request_round_trips_and_a_foreign_message_is_not_a_request() { - let request = HuginnMindRequest::Admit(batch(INSTANCE, vec![identity(INSTANCE)])); + let request = HuginnMindRequest::Admit(PipelineAdmissionBatch { + instance: slug(INSTANCE), + provenance: provenance(Faculty::Hands), + documents: vec![instance(INSTANCE)], + }); let message = encode_request("m-3", &request, Some("eureka-state".into())).unwrap(); assert_eq!(decode_request(&message).unwrap(), ("m-3".into(), request)); diff --git a/crates/huginn-mind/src/lib.rs b/crates/huginn-mind/src/lib.rs index 0fd85e7..b791aac 100644 --- a/crates/huginn-mind/src/lib.rs +++ b/crates/huginn-mind/src/lib.rs @@ -14,7 +14,8 @@ //! selection (CultNet's, over rows this crate supplies) through the same //! `docs` the rules use; nothing is stored for it. //! -//! The wire vocabulary is here too, so the daemon and the client share one set +//! The wire vocabulary and its operation envelope (`envelope`) are here too, +//! so the daemon and the client share one set //! of types. //! //! Document shape, bounds, formats and keys are `epiphany-pipeline`'s; the @@ -26,6 +27,7 @@ pub mod admission; mod docs; +pub mod envelope; pub mod index; pub mod mind; mod rows; From 4f5d0b1a7e622e6d4cb47d6bf080a7c5d162a00b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:32:54 +0200 Subject: [PATCH 026/520] Move the deferred-body bound to the wire vocabulary both sides read MAX_DEFERRED_BODY_BYTES was the daemon's constant, but a client caps its content fetch at the same number, and the client crate cannot depend on the daemon. It moves to huginn-mind::wire next to the response schema it bounds; the daemon imports it, the pin test stays in bodies.rs. Co-Authored-By: Claude Opus 5.5 --- crates/huginn-daemon/src/bodies.rs | 4 +--- crates/huginn-daemon/src/serve.rs | 3 ++- crates/huginn-mind/src/lib.rs | 2 +- crates/huginn-mind/src/wire.rs | 5 +++++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/huginn-daemon/src/bodies.rs b/crates/huginn-daemon/src/bodies.rs index be6de34..2f21fa2 100644 --- a/crates/huginn-daemon/src/bodies.rs +++ b/crates/huginn-daemon/src/bodies.rs @@ -31,9 +31,6 @@ use cultnet_rs::{CultMeshCdnArtifactManifest, CultMeshCdnChunk}; /// The size of one chunk of a deferred body. Four fit one send with room for a /// control-plane answer beside them on the same session. pub const DEFERRED_CHUNK_BYTES: usize = 256 * 1024; -/// The largest answer that is deferred rather than refused, measured on the -/// encoded payload. -pub const MAX_DEFERRED_BODY_BYTES: u64 = 64 * 1024 * 1024; /// What the daemon keeps across all deferred bodies. pub const DEFERRED_BUDGET_BYTES: u64 = 256 * 1024 * 1024; @@ -163,6 +160,7 @@ impl DeferredBodies { mod tests { use super::*; use cultnet_rs::pack_content; + use huginn_mind::MAX_DEFERRED_BODY_BYTES; /// The three numbers a client author is told, spelled out: a chunk of /// 256 KiB, a largest deferred body of 64 MiB, a budget of 256 MiB that diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index bfc54b8..bb9fb8e 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -76,10 +76,11 @@ use huginn_mind::wire::{ use huginn_mind::{Mind, MindRefusal, MindStore, OwnedRedbMessagePackBackingStore}; use crate::bodies::{ - DEFERRED_BUDGET_BYTES, DEFERRED_CHUNK_BYTES, DeferredBodies, MAX_DEFERRED_BODY_BYTES, + DEFERRED_BUDGET_BYTES, DEFERRED_CHUNK_BYTES, DeferredBodies, }; use crate::daemon::{Daemon, Handled, IndexSink, Search, SearchTicket, runtime_id}; use huginn_mind::envelope::{OperationFailure, decode_request, encode_failure, encode_response}; +use huginn_mind::MAX_DEFERRED_BODY_BYTES; use crate::index::{Backoff, Embedder, VectorIndex, WorkerSink}; /// What the loop does when nothing is waiting. A deferred body untouched for diff --git a/crates/huginn-mind/src/lib.rs b/crates/huginn-mind/src/lib.rs index b791aac..29da386 100644 --- a/crates/huginn-mind/src/lib.rs +++ b/crates/huginn-mind/src/lib.rs @@ -52,7 +52,7 @@ pub use receipt::{DocumentVersion, Faculty, HuginnCommitReceipt, PipelineProvena pub use refusal::MindRefusal; pub use store::{MindStore, OwnedRedbMessagePackBackingStore}; pub use wire::{ - DeferredAnswer, HuginnMindRequest, HuginnMindResponse, IndexStatus, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON, + DeferredAnswer, HuginnMindRequest, HuginnMindResponse, IndexStatus, MAX_DEFERRED_BODY_BYTES, MIND_REQUEST_SCHEMA, MIND_REQUEST_SCHEMA_JSON, MIND_RESPONSE_SCHEMA, MIND_RESPONSE_SCHEMA_JSON, MIND_SERVICE_ID, MindStatus, }; diff --git a/crates/huginn-mind/src/wire.rs b/crates/huginn-mind/src/wire.rs index dcbd491..54bce78 100644 --- a/crates/huginn-mind/src/wire.rs +++ b/crates/huginn-mind/src/wire.rs @@ -23,6 +23,11 @@ pub const MIND_SERVICE_ID: &str = "huginn.mind"; pub const MIND_REQUEST_SCHEMA: &str = "huginn.mind_request.v1"; pub const MIND_RESPONSE_SCHEMA: &str = "huginn.mind_response.v1"; +/// The largest answer that is deferred rather than refused, measured on the +/// encoded payload. The daemon refuses past it and a client caps its fetch at +/// it, so both sides read one number. +pub const MAX_DEFERRED_BODY_BYTES: u64 = 64 * 1024 * 1024; + /// The published JSON schema of each wire type, as the catalog advertises it. /// The files are derived from the types below and pinned byte for byte by /// `published_wire_schemas_match_derivation`, so the catalog cannot advertise From 272dc098008d378a6c2cf71a1d1714aed97ec9e1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:41:25 +0200 Subject: [PATCH 027/520] Add the eureka-state client core: HuginnClient::call One call carries one HuginnMindRequest to a daemon over CultNet RUDP on the operation connection and returns the mind's own HuginnMindResponse. Each call is its own session and nothing survives it. A Deferred answer is fetched on that session through cultnet-rs's fetch_content, capped at huginn-mind's MAX_DEFERRED_BODY_BYTES, and decoded, so a caller never sees the deferral. Refusals are answers. ClientError::Unavailable is the only error and names the endpoint. The client opens no store and admits nothing. Tests run a real daemon in-process on a loopback port over a temporary mind, plus scripted servers for the failures a daemon cannot be made to produce. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 +- Cargo.lock | 9 + README.md | 11 +- crates/eureka-state/Cargo.toml | 15 + crates/eureka-state/src/lib.rs | 132 +++++++++ crates/eureka-state/tests/client.rs | 439 ++++++++++++++++++++++++++++ 6 files changed, 605 insertions(+), 7 deletions(-) create mode 100644 crates/eureka-state/tests/client.rs diff --git a/AGENTS.md b/AGENTS.md index e5b0656..bfa7ee0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,12 +18,14 @@ Studio's job, not Huginn's. status included, over `cultcache-rs` and `epiphany-pipeline` pinned by git rev. `huginn-daemon` serves one mind over CultNet RUDP through `cultnet-rs` at the same rev, and reaches the leaf and the store type through - `huginn-mind` so one crate pins each revision. `eureka-state` is a stub. + `huginn-mind` so one crate pins each revision. `eureka-state` is the client + core: one call to a daemon over CultNet, holding no state. - Owned: an instance's memory documents, their admission and the read side that derives their status, in `huginn-mind`, over a redb CultCache store at `/minds//mind.redb`; and their CultNet surface in `huginn-daemon`, which owns the socket, the - sessions, the process and the operation envelope, and no rule. The two wire + sessions and the process, and no rule; the operation envelope is + `huginn-mind`'s. The two wire schemas are published from `schemas/cultnet/`. There is no index yet. - To depend on: Qdrant, directly, once retrieval exists. Unreachable Qdrant is to be a loud refusal, never a fallback store. No crate connects to it yet. diff --git a/Cargo.lock b/Cargo.lock index 1e30f52..f278655 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -335,6 +335,15 @@ dependencies = [ [[package]] name = "eureka-state" version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "cultnet-rs", + "huginn-daemon", + "huginn-mind", + "rmp-serde", + "tempfile", +] [[package]] name = "fastrand" diff --git a/README.md b/README.md index 0b7a07f..985f9d3 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,11 @@ A Cargo workspace of three crates: - `crates/huginn-mind`: storage, identity, admission, and queries and derived status over memory documents. -- `crates/huginn-daemon`: the CultNet surface. The socket, the sessions, the - process and the operation envelope, and no rule. -- `crates/eureka-state`: a stub. It will carry typed state for the Eureka - pipeline; today it holds none. +- `crates/huginn-daemon`: the CultNet surface. The socket, the sessions and + the process, and no rule. The operation envelope both sides speak is + `huginn-mind`'s. +- `crates/eureka-state`: the client core, `HuginnClient::call`: one request to a + daemon over CultNet RUDP, a deferred answer resolved, and no state. `huginn-mind` is live: it opens one instance's store (an owned redb CultCache at `/minds//mind.redb`, locked for the mind's lifetime), @@ -71,7 +72,7 @@ failure and reaches no mind. The two schemas it publishes live in `schemas/cultnet/` and are pinned to their derivation by a test. There is no index yet. -`eureka-state` is a stub; the campaign's cut map in +`eureka-state` has no MCP surface yet; the campaign's cut map in `Epiphany/notes/eureka-pipeline-state-cut.md` owns what each crate must do next. diff --git a/crates/eureka-state/Cargo.toml b/crates/eureka-state/Cargo.toml index 5b49ff0..0de0909 100644 --- a/crates/eureka-state/Cargo.toml +++ b/crates/eureka-state/Cargo.toml @@ -5,4 +5,19 @@ edition.workspace = true license.workspace = true publish.workspace = true +# The client core: one call over CultNet to a Huginn daemon. It owns the +# transport and nothing of the mind. The operation envelope, the wire +# vocabulary and the deferred-body bound are `huginn-mind`'s, so the client and +# the daemon cannot spell them differently; the content fetch and its +# verification are `cultnet-rs`'s. [dependencies] +anyhow = "1" +cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "6251cbcd37c12ef3f0c1cc50063a03369225b7ac", package = "cultnet-rs" } +huginn-mind = { path = "../huginn-mind" } +rmp-serde = "1" + +# Tests run a real daemon in-process against a temporary mind. +[dev-dependencies] +chrono = "0.4.44" +huginn-daemon = { path = "../huginn-daemon" } +tempfile = "3" diff --git a/crates/eureka-state/src/lib.rs b/crates/eureka-state/src/lib.rs index e69de29..15b943d 100644 --- a/crates/eureka-state/src/lib.rs +++ b/crates/eureka-state/src/lib.rs @@ -0,0 +1,132 @@ +//! `eureka-state`, the client core: one call to a Huginn daemon. +//! +//! `HuginnClient::call` sends one `HuginnMindRequest` in CultNet's operation +//! envelope over RUDP, waits for the answer, and returns it as the mind's own +//! `HuginnMindResponse`. An answer too large for one send arrives as a +//! `Deferred` manifest; the client fetches the body on the same session, +//! verifies it, and decodes it, so the caller sees the answer the mind gave +//! and never the deferral. +//! +//! The client owns the transport and no state. A refusal is an answer, so it +//! comes back as `Ok(HuginnMindResponse::Refused(..))`. `ClientError` is the +//! only error, and it means the daemon could not be reached or did not answer +//! in the shape the protocol promises. +//! +//! Every call is its own session: it connects, asks, fetches what was +//! deferred, and drops the session. Nothing is kept between calls. + +use std::fmt; +use std::net::SocketAddr; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result, anyhow, bail}; +use cultnet_rs::{ + CULTNET_OPERATION_CONNECTION_ID, CultMesh, CultMeshRudpSocketOptions, CultNetMessage, + CultNetRudpSocketTransportConnection, fetch_content, +}; +use huginn_mind::envelope::{decode_response, encode_request}; +use huginn_mind::epiphany_pipeline::Slug; +use huginn_mind::{HuginnMindRequest, HuginnMindResponse, MAX_DEFERRED_BODY_BYTES}; + +/// The correlation key of the one request a session carries. +const MESSAGE_ID: &str = "eureka-state-call"; +const POLL: Duration = Duration::from_millis(2); + +/// The daemon could not be reached, went quiet, or did not answer as the +/// protocol says it must. `detail` says which. +#[derive(Debug)] +pub enum ClientError { + Unavailable { endpoint: SocketAddr, detail: String }, +} + +impl fmt::Display for ClientError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self::Unavailable { endpoint, detail } = self; + write!(f, "the Huginn daemon at rudp://{endpoint} is unavailable: {detail}") + } +} + +impl std::error::Error for ClientError {} + +/// A daemon's address, the instance the caller acts for, and how long any one +/// wait may last: the connection, the answer, and each chunk of a deferred +/// body each get `timeout`. +#[derive(Clone, Debug)] +pub struct HuginnClient { + endpoint: SocketAddr, + instance: Slug, + timeout: Duration, +} + +impl HuginnClient { + pub fn new(endpoint: SocketAddr, instance: Slug, timeout: Duration) -> Self { + Self { endpoint, instance, timeout } + } + + /// The instance this client acts for. Requests name their own instance; + /// this is the one the caller's tools fill in. + pub fn instance(&self) -> &Slug { + &self.instance + } + + pub fn call(&self, request: HuginnMindRequest) -> Result { + self.exchange(&request) + .map_err(|error| ClientError::Unavailable { endpoint: self.endpoint, detail: format!("{error:#}") }) + } + + fn exchange(&self, request: &HuginnMindRequest) -> Result { + let mut session = self.connect()?; + let reply = self.ask(&mut session, &encode_request(MESSAGE_ID, request, None)?)?; + let (_, answer) = decode_response(&reply)?; + match answer.map_err(|failure| anyhow!("the daemon rejected the envelope: {}: {}", failure.code, failure.message))? { + HuginnMindResponse::Deferred(deferred) => { + let body = fetch_content(&deferred.manifest, MAX_DEFERRED_BODY_BYTES, |chunk| { + self.ask(&mut session, &chunk) + })?; + match rmp_serde::from_slice::(&body).context("the deferred body is not a response")? { + HuginnMindResponse::Deferred(_) => bail!("a deferred body is itself deferred"), + answered => Ok(answered), + } + } + answered => Ok(answered), + } + } + + fn connect(&self) -> Result { + let mut session = CultMesh::create_rudp_client_for_endpoint( + format!("eureka-state-{}", self.instance.0), + CULTNET_OPERATION_CONNECTION_ID, + &format!("rudp://{}", self.endpoint), + CultMeshRudpSocketOptions::default(), + )?; + session.connect(Vec::new())?; + let deadline = Instant::now() + self.timeout; + loop { + session.poll_resends()?; + let _ = session.receive_once()?; + if session.connected() { + return Ok(session); + } + if Instant::now() >= deadline { + bail!("no accept within {:?}", self.timeout); + } + std::thread::sleep(POLL); + } + } + + /// One message on the session and the next message back, within `timeout`. + fn ask(&self, session: &mut CultNetRudpSocketTransportConnection, message: &CultNetMessage) -> Result { + session.send_schema_message(message)?; + let deadline = Instant::now() + self.timeout; + loop { + session.poll_resends()?; + if let Some(reply) = session.receive_schema_message_once()? { + return Ok(reply); + } + if Instant::now() >= deadline { + bail!("no answer within {:?}", self.timeout); + } + std::thread::sleep(POLL); + } + } +} diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs new file mode 100644 index 0000000..cac6418 --- /dev/null +++ b/crates/eureka-state/tests/client.rs @@ -0,0 +1,439 @@ +//! The client against a real daemon, in-process on a loopback port over a +//! temporary mind, and against scripted servers where the daemon cannot be +//! made to misbehave. + +use std::net::{SocketAddr, UdpSocket}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use chrono::Utc; +use cultnet_rs::{ + CultNetMessage, CultNetRudpServerEvent, CultNetWireContract, Selection, answer_content_chunk_request, + decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, pack_content, +}; +use eureka_state::{ClientError, HuginnClient}; +use huginn_daemon::serve::{MAX_RESPONSE_BYTES, bind, run, schema_registry}; +use huginn_daemon::{Daemon, Handled, Hits, IndexSink, SearchTicket, ServeOptions}; +use huginn_mind::envelope::{OperationFailure, encode_failure, encode_response}; +use huginn_mind::epiphany_pipeline::{ + AuthorityMap, CodeLocation, CutDelete, CutVerification, Date, DocRef, FileChange, Line, NegativeCheck, OrgRepo, + PipelineCampaign, PipelineCutSpec, PipelineDocument, PipelineInstance, PipelineKind, PipelineRef, + PipelineStewardship, Sha, Short, Slug, StructuralDelta, Title, VerificationTest, +}; +use huginn_mind::{ + DeferredAnswer, Faculty, HuginnMindRequest, HuginnMindResponse, IndexStatus, Mind, MindRefusal, MindStore, + OwnedRedbMessagePackBackingStore, PipelineAdmissionBatch, PipelineAdmissionOutcome, PipelineProvenance, +}; +use tempfile::TempDir; + +const INSTANCE: &str = "eureka"; +const OTHER: &str = "thought-cage"; +const CAMPAIGN: &str = "eureka-state"; +const REPO: &str = "GameCult/Epiphany"; +const TIMEOUT: Duration = Duration::from_secs(20); + +fn slug(value: &str) -> Slug { + Slug(value.into()) +} + +fn sha() -> Sha { + Sha("5f98228d".into()) +} + +fn identity(name: &str) -> PipelineDocument { + PipelineDocument::Instance(PipelineInstance { + instance: slug(name), + display_name: Short(format!("{name} mind")), + created_at: Date("2026-09-29".into()), + host: Short(name.into()), + }) +} + +fn stewardship() -> PipelineDocument { + PipelineDocument::Stewardship(PipelineStewardship { + instance: slug(INSTANCE), + repo: OrgRepo(REPO.into()), + sequence: 1, + assigned_on: Date("2026-09-29".into()), + note: "assigned".into(), + }) +} + +fn campaign() -> PipelineDocument { + PipelineDocument::Campaign(PipelineCampaign { + slug: slug(CAMPAIGN), + title: Title("Eureka pipeline state".into()), + repos: vec![OrgRepo(REPO.into())], + working_branch: Short("hands/cut13a".into()), + target_doc: DocRef { path: Short("notes/target.md".into()), start_line: 1, end_line: 9, commit: sha() }, + }) +} + +fn batch(instance: &str, documents: Vec) -> PipelineAdmissionBatch { + PipelineAdmissionBatch { + instance: slug(instance), + provenance: PipelineProvenance { + faculty: Faculty::Hands, + agent: Short("claude".into()), + session: Short("session-1".into()), + tool: Short("admit".into()), + }, + documents, + } +} + +/// A `Line` and a `Short` at the leaf's own bound. +fn line(n: usize) -> Line { + Line(format!("{n:0>1000}")) +} + +fn short(n: usize) -> Short { + Short(format!("{n:0>200}")) +} + +/// A cut spec with every list the leaf bounds filled: at 256 file changes a +/// megabyte of field content, more than one send carries. +fn cut_spec(cut: &str, file_changes: usize) -> PipelineDocument { + let lines = |count: usize| (0..count).map(line).collect::>(); + let shorts = |count: usize| (0..count).map(short).collect::>(); + PipelineDocument::CutSpec(PipelineCutSpec { + campaign: slug(CAMPAIGN), + cut: cut.into(), + revision: 1, + title: Title(short(0).0), + repo: OrgRepo(REPO.into()), + branch: short(1), + base: sha(), + depends_on: vec![], + first: lines(16), + deletes: (0..64).map(|n| CutDelete { path: short(n), lines: 9, note: line(n) }).collect(), + keeps_moves: lines(64), + adds: lines(64), + file_changes: (0..file_changes) + .map(|n| FileChange { + location: CodeLocation { path: short(n), line: 1, end_line: Some(9) }, + change: line(n), + }) + .collect(), + authority_map: Some(AuthorityMap { + owner: line(0), + inputs: lines(16), + outputs: lines(16), + derived_state: lines(16), + forbidden_writers: lines(16), + shared_paths: lines(16), + deletion_line: line(1), + }), + verification: CutVerification { + builds: lines(64), + tests: (0..64).map(|n| VerificationTest { name: short(n), pins: line(n) }).collect(), + negative: (0..64).map(|n| NegativeCheck { pattern: short(n), scope: line(n) }).collect(), + operator: lines(64), + }, + estimate: StructuralDelta { + lines_added: 0, + lines_removed: 0, + dependencies_added: shorts(64), + dependencies_removed: shorts(64), + formats_added: shorts(64), + formats_removed: shorts(64), + targets_added: shorts(64), + targets_removed: shorts(64), + }, + rulings: vec![], + questions: vec![], + }) +} + +fn cut_ref(cut: &str) -> PipelineRef { + PipelineRef { kind: PipelineKind::CutSpec, id: Short(format!("{CAMPAIGN}:cut_spec:cut-{cut}.r1")) } +} + +/// No index: the client's subject is the transport, not the projection. +struct Inert; + +impl IndexSink for Inert { + fn committed(&mut self, _mind: &Mind, _writes: &[PipelineRef]) -> anyhow::Result<()> { + Ok(()) + } + fn status(&self) -> IndexStatus { + IndexStatus::Current + } + fn search(&mut self, _text: &str, _top_k: u32) -> Result { + Err("this index cannot search".into()) + } + fn search_deadline(&self) -> Duration { + Duration::ZERO + } + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { + Vec::new() + } + fn abandon(&mut self, _ticket: SearchTicket) {} +} + +type TestDaemon = Daemon; + +fn answered(handled: Handled) -> HuginnMindResponse { + match handled { + Handled::Answered(response) => response, + Handled::Searching(_) => panic!("the inert index cannot search"), + } +} + +/// A daemon over a temporary mind whose identity is admitted, and whatever +/// else `seed` admits. +fn mind(seed: Vec>) -> (TempDir, TestDaemon) { + let root = tempfile::tempdir().unwrap(); + let mut daemon = Daemon::new(Mind::open(root.path(), &slug(INSTANCE)).unwrap(), Inert); + for documents in std::iter::once(vec![identity(INSTANCE)]).chain(seed) { + let outcome = answered(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), Utc::now())); + assert!(matches!(outcome, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), "{outcome:?}"); + } + (root, daemon) +} + +/// A server on a loopback port that stops when dropped. +struct Server { + addr: SocketAddr, + stopping: Arc, + thread: Option>, + _root: Option, +} + +impl Drop for Server { + fn drop(&mut self) { + self.stopping.store(true, Ordering::Relaxed); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn serve(root: TempDir, mut daemon: TestDaemon) -> Server { + let mut hub = bind("127.0.0.1:0".parse().unwrap(), &daemon.runtime_id()).unwrap(); + let addr = hub.local_addr().unwrap(); + let stopping = Arc::new(AtomicBool::new(false)); + let loop_stopping = Arc::clone(&stopping); + let registry = schema_registry().unwrap(); + let thread = std::thread::spawn(move || { + run(&mut daemon, &mut hub, ®istry, &loop_stopping, &ServeOptions::default()).unwrap(); + }); + Server { addr, stopping, thread: Some(thread), _root: Some(root) } +} + +/// A server that answers each schema message with whatever `script` says. +fn scripted(mut script: impl FnMut(&CultNetMessage) -> CultNetMessage + Send + 'static) -> Server { + let mut hub = bind("127.0.0.1:0".parse().unwrap(), "scripted").unwrap(); + let addr = hub.local_addr().unwrap(); + let stopping = Arc::new(AtomicBool::new(false)); + let loop_stopping = Arc::clone(&stopping); + let thread = std::thread::spawn(move || { + while !loop_stopping.load(Ordering::Relaxed) { + hub.poll_resends().unwrap(); + while let Some(event) = hub.receive_event_once().unwrap() { + let CultNetRudpServerEvent::Frame { session, frame } = event else { continue }; + let message = + decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0).unwrap(); + hub.send_schema_message(&session, &script(&message)).unwrap(); + } + std::thread::sleep(Duration::from_millis(2)); + } + }); + Server { addr, stopping, thread: Some(thread), _root: None } +} + +/// A scripted server that answers any operation with a deferral of `body` and +/// serves its chunks faithfully. +fn deferring(body: Vec) -> Server { + let (manifest, chunks) = pack_content("art", "package", "", "", "", &body, 256 * 1024).expect("a body packs"); + let deferred = encode_response( + "eureka-state-call", + "whoami", + &HuginnMindResponse::Deferred(DeferredAnswer { manifest }), + "scripted", + ) + .unwrap(); + scripted(move |message| match message { + CultNetMessage::OperationRequest { .. } => deferred.clone(), + chunk => answer_content_chunk_request(chunk, |hash| { + chunks.iter().find(|c| c.chunk_hash == hash).map(|c| c.payload.as_slice()) + }), + }) +} + +fn client(addr: SocketAddr) -> HuginnClient { + HuginnClient::new(addr, slug(INSTANCE), TIMEOUT) +} + +fn unavailable(error: ClientError) -> (SocketAddr, String) { + let ClientError::Unavailable { endpoint, detail } = error; + (endpoint, detail) +} + +fn document_query(instance: &str) -> HuginnMindRequest { + HuginnMindRequest::Query { + instance: slug(instance), + selection: Selection { projection: "document".into(), ..Selection::default() }, + semantic: None, + } +} + +/// Every operation the wire has, asked of a live daemon over the socket, and +/// each answered with the mind's own answer. +#[test] +fn every_operation_round_trips_against_a_live_daemon() { + let (root, daemon) = mind(vec![]); + let server = serve(root, daemon); + let client = client(server.addr); + + let HuginnMindResponse::Whoami(status) = client.call(HuginnMindRequest::Whoami).unwrap() else { + panic!("whoami is answered with a status"); + }; + assert_eq!((status.instance, status.documents), (slug(INSTANCE), 1)); + + let admitted = client.call(HuginnMindRequest::Admit(batch(INSTANCE, vec![stewardship(), campaign()]))).unwrap(); + let HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { writes, .. }) = admitted else { + panic!("the batch is committed, got {admitted:?}"); + }; + assert_eq!(writes.len(), 2); + + let HuginnMindResponse::Query(page) = client.call(document_query(INSTANCE)).unwrap() else { + panic!("a query is answered with a page"); + }; + assert_eq!(page.matched, 3, "the identity and the two admitted documents"); + + let campaign_ref = PipelineRef { kind: PipelineKind::Campaign, id: Short(format!("{CAMPAIGN}:campaign:self")) }; + let viewed = client.call(HuginnMindRequest::View { instance: slug(INSTANCE), id: campaign_ref.clone() }).unwrap(); + let HuginnMindResponse::View(Some(view)) = viewed else { panic!("the campaign is viewable, got {viewed:?}") }; + assert_eq!(view.id, campaign_ref); +} + +/// A refusal is data. A read that names another mind, a write that declares +/// another mind, and a document the mind does not hold each come back as `Ok`. +#[test] +fn a_foreign_instance_is_an_answer_not_an_error() { + let (root, daemon) = mind(vec![]); + let server = serve(root, daemon); + let client = client(server.addr); + let foreign = MindRefusal::ForeignInstance { declared: OTHER.into(), mind: INSTANCE.into() }; + + assert_eq!(client.call(document_query(OTHER)).unwrap(), HuginnMindResponse::Refused(foreign.clone())); + assert_eq!( + client.call(HuginnMindRequest::Admit(batch(OTHER, vec![identity(OTHER)]))).unwrap(), + HuginnMindResponse::Admit(PipelineAdmissionOutcome::Refused(foreign)), + ); + let absent = PipelineRef { kind: PipelineKind::CutSpec, id: Short(format!("{CAMPAIGN}:cut_spec:cut-none.r1")) }; + assert_eq!( + client.call(HuginnMindRequest::View { instance: slug(INSTANCE), id: absent }).unwrap(), + HuginnMindResponse::View(None), + ); +} + +/// The wide cut spec's answer is over one send and arrives deferred; the +/// narrow one arrives whole. Both come back equal to what the mind gave. +#[test] +fn a_deferred_answer_resolves_to_the_same_page_as_a_direct_one() { + let (root, mut daemon) = + mind(vec![vec![stewardship(), campaign()], vec![cut_spec("wide", 256), cut_spec("fits", 180)]]); + let measured = |request: &HuginnMindRequest, daemon: &mut TestDaemon| { + let answer = answered(daemon.handle(request.clone(), Utc::now())); + let envelope = encode_response("m", request.operation(), &answer, "huginn-eureka").unwrap(); + let size = encode_cultnet_message_to_vec(&envelope, CultNetWireContract::CultNetSchemaV0).unwrap().len(); + (answer, size as u64) + }; + let wide = HuginnMindRequest::View { instance: slug(INSTANCE), id: cut_ref("wide") }; + let fits = HuginnMindRequest::View { instance: slug(INSTANCE), id: cut_ref("fits") }; + let (wide_answer, wide_size) = measured(&wide, &mut daemon); + let (fits_answer, fits_size) = measured(&fits, &mut daemon); + assert!(matches!(wide_answer, HuginnMindResponse::View(Some(_))), "{wide_answer:?}"); + assert!(wide_size > MAX_RESPONSE_BYTES, "the wide answer must be over one send, or nothing was deferred"); + assert!(fits_size <= MAX_RESPONSE_BYTES, "the narrow answer must fit one send"); + + let server = serve(root, daemon); + let client = client(server.addr); + assert_eq!(client.call(wide).unwrap(), wide_answer); + assert_eq!(client.call(fits).unwrap(), fits_answer); +} + +/// Nothing listens on the port. The call fails naming the endpoint, and it +/// fails when the configured timeout says: two timeouts, two waits. +#[test] +fn a_closed_port_is_unavailable_naming_the_endpoint_within_the_timeout() { + let closed = UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap(); + let mut waited = Vec::new(); + for timeout in [Duration::from_millis(300), Duration::from_millis(1200)] { + let started = Instant::now(); + let error = HuginnClient::new(closed, slug(INSTANCE), timeout).call(HuginnMindRequest::Whoami).unwrap_err(); + let elapsed = started.elapsed(); + assert!(elapsed >= timeout, "gave up after {elapsed:?}, before its {timeout:?}"); + assert!(elapsed < timeout + Duration::from_millis(800), "gave up after {elapsed:?}, long past its {timeout:?}"); + assert!(error.to_string().contains(&closed.to_string()), "{error}"); + let (endpoint, detail) = unavailable(error); + assert_eq!(endpoint, closed); + assert!(detail.contains("no accept"), "{detail}"); + waited.push(elapsed); + } + assert!(waited[1] > waited[0] + Duration::from_millis(500), "{waited:?}"); +} + +/// A daemon that accepts and then says nothing is a timeout on the answer, +/// not on the connection. +#[test] +fn a_daemon_that_never_answers_is_unavailable_within_the_timeout() { + let server = scripted(|_| { + std::thread::sleep(Duration::from_secs(3)); + CultNetMessage::Error { error: "late".into(), code: None, details: None } + }); + let started = Instant::now(); + let error = HuginnClient::new(server.addr, slug(INSTANCE), Duration::from_millis(400)) + .call(HuginnMindRequest::Whoami) + .unwrap_err(); + let elapsed = started.elapsed(); + assert!(elapsed >= Duration::from_millis(400) && elapsed < Duration::from_secs(2), "{elapsed:?}"); + let (endpoint, detail) = unavailable(error); + assert_eq!(endpoint, server.addr); + assert!(detail.contains("no answer"), "{detail}"); +} + +/// An envelope failure means the request never reached a mind; it is not an +/// answer, so the caller sees the code. +#[test] +fn an_envelope_failure_is_unavailable_and_carries_its_code() { + let server = scripted(|_| { + encode_failure( + "eureka-state-call", + "whoami", + &OperationFailure { code: "wrong-service".into(), message: "not huginn.mind".into() }, + "scripted", + ) + }); + let (_, detail) = unavailable(client(server.addr).call(HuginnMindRequest::Whoami).unwrap_err()); + assert!(detail.contains("wrong-service"), "{detail}"); +} + +/// A deferred body that is not a response, and one that is another deferral, +/// are protocol violations named as such. +#[test] +fn a_deferred_body_must_be_a_response_and_not_another_deferral() { + let garbage = deferring(vec![0xc1; 40_000]); + let (_, detail) = unavailable(client(garbage.addr).call(HuginnMindRequest::Whoami).unwrap_err()); + assert!(detail.contains("not a response"), "{detail}"); + + let (inner, _) = pack_content("inner", "package", "", "", "", b"x", 16).unwrap(); + let again = rmp_serde::to_vec_named(&HuginnMindResponse::Deferred(DeferredAnswer { manifest: inner })).unwrap(); + let nested = deferring(again); + let (_, detail) = unavailable(client(nested.addr).call(HuginnMindRequest::Whoami).unwrap_err()); + assert!(detail.contains("itself deferred"), "{detail}"); +} + +/// A body over the deferral cap is refused before its first chunk is asked +/// for, by the number the daemon defers under. +#[test] +fn a_body_over_the_deferral_cap_is_refused() { + let over = huginn_mind::MAX_DEFERRED_BODY_BYTES as usize + 1; + let server = deferring(vec![0_u8; over]); + let (_, detail) = unavailable(client(server.addr).call(HuginnMindRequest::Whoami).unwrap_err()); + assert!(detail.contains("exceeds"), "{detail}"); +} From 26f4137d55f86735ae934487740135a64042e79e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:46:33 +0200 Subject: [PATCH 028/520] Pin the client's configured instance against a mutation that replaces it Co-Authored-By: Claude Opus 5.5 --- crates/eureka-state/tests/client.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs index cac6418..78519bb 100644 --- a/crates/eureka-state/tests/client.rs +++ b/crates/eureka-state/tests/client.rs @@ -287,6 +287,7 @@ fn every_operation_round_trips_against_a_live_daemon() { let (root, daemon) = mind(vec![]); let server = serve(root, daemon); let client = client(server.addr); + assert_eq!(client.instance(), &slug(INSTANCE), "the client reports the instance it was configured for"); let HuginnMindResponse::Whoami(status) = client.call(HuginnMindRequest::Whoami).unwrap() else { panic!("whoami is answered with a status"); From 3dc50eec8e23018827c31d90dbfd4fd675f9cf65 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:16:33 +0200 Subject: [PATCH 029/520] eureka-state: whole-call deadline, correlated answers, Rejected, disconnect The timeout now bounds the whole call: one deadline at call start, every wait gets what remains, so a chunk-by-chunk deferred body cannot outlive it. The response's message id and operation are checked against the request. An envelope-level OperationFailure is ClientError::Rejected, distinct from the transport's Unavailable, so a retry on Unavailable cannot loop on a permanent protocol mismatch. Every call disconnects its session, success or failure. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/src/lib.rs | 103 ++++++++++++++----- crates/eureka-state/tests/client.rs | 150 ++++++++++++++++++++++++---- 2 files changed, 206 insertions(+), 47 deletions(-) diff --git a/crates/eureka-state/src/lib.rs b/crates/eureka-state/src/lib.rs index 15b943d..94dbfec 100644 --- a/crates/eureka-state/src/lib.rs +++ b/crates/eureka-state/src/lib.rs @@ -13,7 +13,8 @@ //! in the shape the protocol promises. //! //! Every call is its own session: it connects, asks, fetches what was -//! deferred, and drops the session. Nothing is kept between calls. +//! deferred, and disconnects, whether the call succeeded or failed. Nothing is +//! kept between calls. use std::fmt; use std::net::SocketAddr; @@ -24,7 +25,7 @@ use cultnet_rs::{ CULTNET_OPERATION_CONNECTION_ID, CultMesh, CultMeshRudpSocketOptions, CultNetMessage, CultNetRudpSocketTransportConnection, fetch_content, }; -use huginn_mind::envelope::{decode_response, encode_request}; +use huginn_mind::envelope::{OperationFailure, decode_response, encode_request}; use huginn_mind::epiphany_pipeline::Slug; use huginn_mind::{HuginnMindRequest, HuginnMindResponse, MAX_DEFERRED_BODY_BYTES}; @@ -32,25 +33,47 @@ use huginn_mind::{HuginnMindRequest, HuginnMindResponse, MAX_DEFERRED_BODY_BYTES const MESSAGE_ID: &str = "eureka-state-call"; const POLL: Duration = Duration::from_millis(2); -/// The daemon could not be reached, went quiet, or did not answer as the -/// protocol says it must. `detail` says which. +/// Why a call produced no answer. `Unavailable` is the transport's failure and +/// the only one worth retrying: the daemon could not be reached, went quiet +/// past the call's deadline, or answered out of protocol. `Rejected` is the +/// daemon refusing the envelope itself, which a retry of the same call cannot +/// change. #[derive(Debug)] pub enum ClientError { Unavailable { endpoint: SocketAddr, detail: String }, + Rejected { endpoint: SocketAddr, code: String, detail: String }, } impl fmt::Display for ClientError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Self::Unavailable { endpoint, detail } = self; - write!(f, "the Huginn daemon at rudp://{endpoint} is unavailable: {detail}") + match self { + Self::Unavailable { endpoint, detail } => { + write!(f, "the Huginn daemon at rudp://{endpoint} is unavailable: {detail}") + } + Self::Rejected { endpoint, code, detail } => { + write!(f, "the Huginn daemon at rudp://{endpoint} rejected the request: {code}: {detail}") + } + } } } impl std::error::Error for ClientError {} -/// A daemon's address, the instance the caller acts for, and how long any one -/// wait may last: the connection, the answer, and each chunk of a deferred -/// body each get `timeout`. +/// What `exchange` can fail with, before the endpoint is attached. +enum Failure { + Transport(anyhow::Error), + Rejected(OperationFailure), +} + +impl From for Failure { + fn from(error: anyhow::Error) -> Self { + Self::Transport(error) + } +} + +/// A daemon's address, the instance the caller acts for, and the time one whole +/// call may take: the connection, the answer, and every chunk of a deferred +/// body together must finish within `timeout` of the call starting. #[derive(Clone, Debug)] pub struct HuginnClient { endpoint: SocketAddr, @@ -70,21 +93,47 @@ impl HuginnClient { } pub fn call(&self, request: HuginnMindRequest) -> Result { - self.exchange(&request) - .map_err(|error| ClientError::Unavailable { endpoint: self.endpoint, detail: format!("{error:#}") }) + let deadline = Instant::now() + self.timeout; + let unavailable = + |error: anyhow::Error| ClientError::Unavailable { endpoint: self.endpoint, detail: format!("{error:#}") }; + let mut session = self.connect(deadline).map_err(unavailable)?; + let outcome = self.exchange(&mut session, &request, deadline); + // Best effort: the daemon drops a session it never hears from, only later. + let _ = session.disconnect(Vec::new()); + outcome.map_err(|failure| match failure { + Failure::Transport(error) => unavailable(error), + Failure::Rejected(rejection) => ClientError::Rejected { + endpoint: self.endpoint, + code: rejection.code, + detail: rejection.message, + }, + }) } - fn exchange(&self, request: &HuginnMindRequest) -> Result { - let mut session = self.connect()?; - let reply = self.ask(&mut session, &encode_request(MESSAGE_ID, request, None)?)?; - let (_, answer) = decode_response(&reply)?; - match answer.map_err(|failure| anyhow!("the daemon rejected the envelope: {}: {}", failure.code, failure.message))? { + fn exchange( + &self, + session: &mut CultNetRudpSocketTransportConnection, + request: &HuginnMindRequest, + deadline: Instant, + ) -> Result { + let reply = self.ask(session, &encode_request(MESSAGE_ID, request, None)?, deadline)?; + let CultNetMessage::OperationResponse { operation, .. } = &reply else { + return Err(anyhow!("the daemon answered with something other than an operation response").into()); + }; + if operation != request.operation() { + return Err(anyhow!("the daemon answered {operation}, not the {} that was asked", request.operation()).into()); + } + let (message_id, answer) = decode_response(&reply)?; + if message_id != MESSAGE_ID { + return Err(anyhow!("the daemon answered request {message_id}, not {MESSAGE_ID}").into()); + } + match answer.map_err(Failure::Rejected)? { HuginnMindResponse::Deferred(deferred) => { let body = fetch_content(&deferred.manifest, MAX_DEFERRED_BODY_BYTES, |chunk| { - self.ask(&mut session, &chunk) + self.ask(session, &chunk, deadline) })?; match rmp_serde::from_slice::(&body).context("the deferred body is not a response")? { - HuginnMindResponse::Deferred(_) => bail!("a deferred body is itself deferred"), + HuginnMindResponse::Deferred(_) => Err(anyhow!("a deferred body is itself deferred").into()), answered => Ok(answered), } } @@ -92,7 +141,7 @@ impl HuginnClient { } } - fn connect(&self) -> Result { + fn connect(&self, deadline: Instant) -> Result { let mut session = CultMesh::create_rudp_client_for_endpoint( format!("eureka-state-{}", self.instance.0), CULTNET_OPERATION_CONNECTION_ID, @@ -100,7 +149,6 @@ impl HuginnClient { CultMeshRudpSocketOptions::default(), )?; session.connect(Vec::new())?; - let deadline = Instant::now() + self.timeout; loop { session.poll_resends()?; let _ = session.receive_once()?; @@ -108,23 +156,28 @@ impl HuginnClient { return Ok(session); } if Instant::now() >= deadline { - bail!("no accept within {:?}", self.timeout); + bail!("no accept before the call's {:?} deadline", self.timeout); } std::thread::sleep(POLL); } } - /// One message on the session and the next message back, within `timeout`. - fn ask(&self, session: &mut CultNetRudpSocketTransportConnection, message: &CultNetMessage) -> Result { + /// One message on the session and the next message back, before the call's + /// `deadline`. + fn ask( + &self, + session: &mut CultNetRudpSocketTransportConnection, + message: &CultNetMessage, + deadline: Instant, + ) -> Result { session.send_schema_message(message)?; - let deadline = Instant::now() + self.timeout; loop { session.poll_resends()?; if let Some(reply) = session.receive_schema_message_once()? { return Ok(reply); } if Instant::now() >= deadline { - bail!("no answer within {:?}", self.timeout); + bail!("no answer before the call's {:?} deadline", self.timeout); } std::thread::sleep(POLL); } diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs index 78519bb..07f458a 100644 --- a/crates/eureka-state/tests/client.rs +++ b/crates/eureka-state/tests/client.rs @@ -4,7 +4,7 @@ use std::net::{SocketAddr, UdpSocket}; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; @@ -199,6 +199,9 @@ struct Server { addr: SocketAddr, stopping: Arc, thread: Option>, + /// The sessions a scripted server has seen end: what the client's + /// disconnect looks like from the daemon's side. + ended: Arc, _root: Option, } @@ -220,7 +223,7 @@ fn serve(root: TempDir, mut daemon: TestDaemon) -> Server { let thread = std::thread::spawn(move || { run(&mut daemon, &mut hub, ®istry, &loop_stopping, &ServeOptions::default()).unwrap(); }); - Server { addr, stopping, thread: Some(thread), _root: Some(root) } + Server { addr, stopping, thread: Some(thread), ended: Arc::default(), _root: Some(root) } } /// A server that answers each schema message with whatever `script` says. @@ -229,10 +232,15 @@ fn scripted(mut script: impl FnMut(&CultNetMessage) -> CultNetMessage + Send + ' let addr = hub.local_addr().unwrap(); let stopping = Arc::new(AtomicBool::new(false)); let loop_stopping = Arc::clone(&stopping); + let ended = Arc::new(AtomicUsize::new(0)); + let loop_ended = Arc::clone(&ended); let thread = std::thread::spawn(move || { while !loop_stopping.load(Ordering::Relaxed) { hub.poll_resends().unwrap(); while let Some(event) = hub.receive_event_once().unwrap() { + if matches!(event, CultNetRudpServerEvent::Disconnected { .. }) { + loop_ended.fetch_add(1, Ordering::Relaxed); + } let CultNetRudpServerEvent::Frame { session, frame } = event else { continue }; let message = decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0).unwrap(); @@ -241,13 +249,19 @@ fn scripted(mut script: impl FnMut(&CultNetMessage) -> CultNetMessage + Send + ' std::thread::sleep(Duration::from_millis(2)); } }); - Server { addr, stopping, thread: Some(thread), _root: None } + Server { addr, stopping, thread: Some(thread), ended, _root: None } } /// A scripted server that answers any operation with a deferral of `body` and /// serves its chunks faithfully. fn deferring(body: Vec) -> Server { - let (manifest, chunks) = pack_content("art", "package", "", "", "", &body, 256 * 1024).expect("a body packs"); + deferring_in_chunks(body, 256 * 1024, Duration::ZERO) +} + +/// `deferring`, with `chunk_bytes` per chunk and each chunk answered only after +/// `drip`. +fn deferring_in_chunks(body: Vec, chunk_bytes: usize, drip: Duration) -> Server { + let (manifest, chunks) = pack_content("art", "package", "", "", "", &body, chunk_bytes).expect("a body packs"); let deferred = encode_response( "eureka-state-call", "whoami", @@ -257,9 +271,12 @@ fn deferring(body: Vec) -> Server { .unwrap(); scripted(move |message| match message { CultNetMessage::OperationRequest { .. } => deferred.clone(), - chunk => answer_content_chunk_request(chunk, |hash| { - chunks.iter().find(|c| c.chunk_hash == hash).map(|c| c.payload.as_slice()) - }), + chunk => { + std::thread::sleep(drip); + answer_content_chunk_request(chunk, |hash| { + chunks.iter().find(|c| c.chunk_hash == hash).map(|c| c.payload.as_slice()) + }) + } }) } @@ -268,8 +285,38 @@ fn client(addr: SocketAddr) -> HuginnClient { } fn unavailable(error: ClientError) -> (SocketAddr, String) { - let ClientError::Unavailable { endpoint, detail } = error; - (endpoint, detail) + match error { + ClientError::Unavailable { endpoint, detail } => (endpoint, detail), + rejected @ ClientError::Rejected { .. } => panic!("expected Unavailable, got {rejected}"), + } +} + +fn rejected(error: ClientError) -> (SocketAddr, String, String) { + match error { + ClientError::Rejected { endpoint, code, detail } => (endpoint, code, detail), + unavailable @ ClientError::Unavailable { .. } => panic!("expected Rejected, got {unavailable}"), + } +} + +/// A scripted server whose every answer is `reply`. +fn answering(reply: CultNetMessage) -> Server { + scripted(move |_| reply.clone()) +} + +fn nothing_found(message_id: &str, operation: &str) -> CultNetMessage { + encode_response(message_id, operation, &HuginnMindResponse::View(None), "scripted").unwrap() +} + +/// Wait for a scripted server to have seen `count` sessions end. +fn ended_within(server: &Server, count: usize, wait: Duration) -> bool { + let until = Instant::now() + wait; + while server.ended.load(Ordering::Relaxed) < count { + if Instant::now() >= until { + return false; + } + std::thread::sleep(Duration::from_millis(10)); + } + true } fn document_query(instance: &str) -> HuginnMindRequest { @@ -373,7 +420,7 @@ fn a_closed_port_is_unavailable_naming_the_endpoint_within_the_timeout() { assert!(error.to_string().contains(&closed.to_string()), "{error}"); let (endpoint, detail) = unavailable(error); assert_eq!(endpoint, closed); - assert!(detail.contains("no accept"), "{detail}"); + assert!(detail.contains("no accept") && detail.contains("deadline"), "{detail}"); waited.push(elapsed); } assert!(waited[1] > waited[0] + Duration::from_millis(500), "{waited:?}"); @@ -398,20 +445,79 @@ fn a_daemon_that_never_answers_is_unavailable_within_the_timeout() { assert!(detail.contains("no answer"), "{detail}"); } -/// An envelope failure means the request never reached a mind; it is not an -/// answer, so the caller sees the code. +/// An envelope failure means the request never reached a mind. It is the +/// daemon refusing, not a transport failure, so a caller that retries on +/// `Unavailable` does not loop on it; the code and the message arrive intact. #[test] -fn an_envelope_failure_is_unavailable_and_carries_its_code() { - let server = scripted(|_| { - encode_failure( - "eureka-state-call", - "whoami", - &OperationFailure { code: "wrong-service".into(), message: "not huginn.mind".into() }, - "scripted", - ) +fn an_envelope_failure_is_rejected_and_carries_its_code() { + let server = answering(encode_failure( + "eureka-state-call", + "whoami", + &OperationFailure { code: "wrong-service".into(), message: "not huginn.mind".into() }, + "scripted", + )); + let error = client(server.addr).call(HuginnMindRequest::Whoami).unwrap_err(); + assert!(error.to_string().contains("wrong-service"), "{error}"); + let (endpoint, code, detail) = rejected(error); + assert_eq!((endpoint, code.as_str(), detail.as_str()), (server.addr, "wrong-service", "not huginn.mind")); +} + +/// An answer to another request, or to another operation, is not this call's +/// answer, whatever it says. +#[test] +fn an_answer_that_is_not_to_this_request_is_refused() { + let foreign = answering(nothing_found("someone-elses-call", "whoami")); + let (_, detail) = unavailable(client(foreign.addr).call(HuginnMindRequest::Whoami).unwrap_err()); + assert!(detail.contains("someone-elses-call"), "{detail}"); + + let other_operation = answering(nothing_found("eureka-state-call", "query")); + let (_, detail) = unavailable(client(other_operation.addr).call(HuginnMindRequest::Whoami).unwrap_err()); + assert!(detail.contains("query") && detail.contains("whoami"), "{detail}"); + + let matching = answering(nothing_found("eureka-state-call", "whoami")); + assert_eq!(client(matching.addr).call(HuginnMindRequest::Whoami).unwrap(), HuginnMindResponse::View(None)); +} + +/// The timeout bounds the whole call. Seven chunks each answered after 300 ms +/// take two seconds; no single wait exceeds the 500 ms timeout, and the call +/// still gives up at it. +#[test] +fn the_timeout_bounds_the_whole_call_not_each_wait() { + let server = deferring_in_chunks(vec![7_u8; 7 * 1024], 1024, Duration::from_millis(300)); + let timeout = Duration::from_millis(500); + let started = Instant::now(); + let error = HuginnClient::new(server.addr, slug(INSTANCE), timeout).call(HuginnMindRequest::Whoami).unwrap_err(); + let elapsed = started.elapsed(); + assert!(elapsed >= timeout && elapsed < Duration::from_millis(1100), "{elapsed:?}"); + let (_, detail) = unavailable(error); + assert!(detail.contains("deadline"), "{detail}"); +} + +/// Every call ends its session, whether it answered, was rejected, or gave up. +#[test] +fn every_call_disconnects_its_session() { + let answered = answering(nothing_found("eureka-state-call", "whoami")); + client(answered.addr).call(HuginnMindRequest::Whoami).unwrap(); + assert!(ended_within(&answered, 1, Duration::from_secs(5)), "an answered call left its session open"); + + let refused = answering(encode_failure( + "eureka-state-call", + "whoami", + &OperationFailure { code: "wrong-service".into(), message: "no".into() }, + "scripted", + )); + rejected(client(refused.addr).call(HuginnMindRequest::Whoami).unwrap_err()); + assert!(ended_within(&refused, 1, Duration::from_secs(5)), "a rejected call left its session open"); + + let silent = scripted(|_| { + std::thread::sleep(Duration::from_millis(800)); + CultNetMessage::Error { error: "late".into(), code: None, details: None } }); - let (_, detail) = unavailable(client(server.addr).call(HuginnMindRequest::Whoami).unwrap_err()); - assert!(detail.contains("wrong-service"), "{detail}"); + let error = HuginnClient::new(silent.addr, slug(INSTANCE), Duration::from_millis(300)) + .call(HuginnMindRequest::Whoami) + .unwrap_err(); + unavailable(error); + assert!(ended_within(&silent, 1, Duration::from_secs(5)), "a timed-out call left its session open"); } /// A deferred body that is not a response, and one that is another deferral, From 8041d82fe1d65fbdbb05d9080be217c5f2e56108 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:36:18 +0200 Subject: [PATCH 030/520] eureka-state tests: the in-process daemon fixtures move to tests/common The daemon half of the client's fixtures (an in-process daemon over a temporary mind on a loopback port, and the scripted server) becomes tests/common/mod.rs, so the MCP binary's tests (Cut 13b) drive the same daemon instead of a second copy of it. No behaviour change; client.rs shrinks by the moved lines. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/tests/client.rs | 153 +-------------------- crates/eureka-state/tests/common/mod.rs | 174 ++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 150 deletions(-) create mode 100644 crates/eureka-state/tests/common/mod.rs diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs index 07f458a..1ef33e8 100644 --- a/crates/eureka-state/tests/client.rs +++ b/crates/eureka-state/tests/client.rs @@ -28,29 +28,12 @@ use huginn_mind::{ }; use tempfile::TempDir; -const INSTANCE: &str = "eureka"; +mod common; +use common::*; + const OTHER: &str = "thought-cage"; -const CAMPAIGN: &str = "eureka-state"; -const REPO: &str = "GameCult/Epiphany"; const TIMEOUT: Duration = Duration::from_secs(20); -fn slug(value: &str) -> Slug { - Slug(value.into()) -} - -fn sha() -> Sha { - Sha("5f98228d".into()) -} - -fn identity(name: &str) -> PipelineDocument { - PipelineDocument::Instance(PipelineInstance { - instance: slug(name), - display_name: Short(format!("{name} mind")), - created_at: Date("2026-09-29".into()), - host: Short(name.into()), - }) -} - fn stewardship() -> PipelineDocument { PipelineDocument::Stewardship(PipelineStewardship { instance: slug(INSTANCE), @@ -61,29 +44,6 @@ fn stewardship() -> PipelineDocument { }) } -fn campaign() -> PipelineDocument { - PipelineDocument::Campaign(PipelineCampaign { - slug: slug(CAMPAIGN), - title: Title("Eureka pipeline state".into()), - repos: vec![OrgRepo(REPO.into())], - working_branch: Short("hands/cut13a".into()), - target_doc: DocRef { path: Short("notes/target.md".into()), start_line: 1, end_line: 9, commit: sha() }, - }) -} - -fn batch(instance: &str, documents: Vec) -> PipelineAdmissionBatch { - PipelineAdmissionBatch { - instance: slug(instance), - provenance: PipelineProvenance { - faculty: Faculty::Hands, - agent: Short("claude".into()), - session: Short("session-1".into()), - tool: Short("admit".into()), - }, - documents, - } -} - /// A `Line` and a `Short` at the leaf's own bound. fn line(n: usize) -> Line { Line(format!("{n:0>1000}")) @@ -151,108 +111,6 @@ fn cut_ref(cut: &str) -> PipelineRef { PipelineRef { kind: PipelineKind::CutSpec, id: Short(format!("{CAMPAIGN}:cut_spec:cut-{cut}.r1")) } } -/// No index: the client's subject is the transport, not the projection. -struct Inert; - -impl IndexSink for Inert { - fn committed(&mut self, _mind: &Mind, _writes: &[PipelineRef]) -> anyhow::Result<()> { - Ok(()) - } - fn status(&self) -> IndexStatus { - IndexStatus::Current - } - fn search(&mut self, _text: &str, _top_k: u32) -> Result { - Err("this index cannot search".into()) - } - fn search_deadline(&self) -> Duration { - Duration::ZERO - } - fn searched(&mut self) -> Vec<(SearchTicket, Result)> { - Vec::new() - } - fn abandon(&mut self, _ticket: SearchTicket) {} -} - -type TestDaemon = Daemon; - -fn answered(handled: Handled) -> HuginnMindResponse { - match handled { - Handled::Answered(response) => response, - Handled::Searching(_) => panic!("the inert index cannot search"), - } -} - -/// A daemon over a temporary mind whose identity is admitted, and whatever -/// else `seed` admits. -fn mind(seed: Vec>) -> (TempDir, TestDaemon) { - let root = tempfile::tempdir().unwrap(); - let mut daemon = Daemon::new(Mind::open(root.path(), &slug(INSTANCE)).unwrap(), Inert); - for documents in std::iter::once(vec![identity(INSTANCE)]).chain(seed) { - let outcome = answered(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), Utc::now())); - assert!(matches!(outcome, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), "{outcome:?}"); - } - (root, daemon) -} - -/// A server on a loopback port that stops when dropped. -struct Server { - addr: SocketAddr, - stopping: Arc, - thread: Option>, - /// The sessions a scripted server has seen end: what the client's - /// disconnect looks like from the daemon's side. - ended: Arc, - _root: Option, -} - -impl Drop for Server { - fn drop(&mut self) { - self.stopping.store(true, Ordering::Relaxed); - if let Some(thread) = self.thread.take() { - let _ = thread.join(); - } - } -} - -fn serve(root: TempDir, mut daemon: TestDaemon) -> Server { - let mut hub = bind("127.0.0.1:0".parse().unwrap(), &daemon.runtime_id()).unwrap(); - let addr = hub.local_addr().unwrap(); - let stopping = Arc::new(AtomicBool::new(false)); - let loop_stopping = Arc::clone(&stopping); - let registry = schema_registry().unwrap(); - let thread = std::thread::spawn(move || { - run(&mut daemon, &mut hub, ®istry, &loop_stopping, &ServeOptions::default()).unwrap(); - }); - Server { addr, stopping, thread: Some(thread), ended: Arc::default(), _root: Some(root) } -} - -/// A server that answers each schema message with whatever `script` says. -fn scripted(mut script: impl FnMut(&CultNetMessage) -> CultNetMessage + Send + 'static) -> Server { - let mut hub = bind("127.0.0.1:0".parse().unwrap(), "scripted").unwrap(); - let addr = hub.local_addr().unwrap(); - let stopping = Arc::new(AtomicBool::new(false)); - let loop_stopping = Arc::clone(&stopping); - let ended = Arc::new(AtomicUsize::new(0)); - let loop_ended = Arc::clone(&ended); - let thread = std::thread::spawn(move || { - while !loop_stopping.load(Ordering::Relaxed) { - hub.poll_resends().unwrap(); - while let Some(event) = hub.receive_event_once().unwrap() { - if matches!(event, CultNetRudpServerEvent::Disconnected { .. }) { - loop_ended.fetch_add(1, Ordering::Relaxed); - } - let CultNetRudpServerEvent::Frame { session, frame } = event else { continue }; - let message = - decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0).unwrap(); - hub.send_schema_message(&session, &script(&message)).unwrap(); - } - std::thread::sleep(Duration::from_millis(2)); - } - }); - Server { addr, stopping, thread: Some(thread), ended, _root: None } -} - -/// A scripted server that answers any operation with a deferral of `body` and /// serves its chunks faithfully. fn deferring(body: Vec) -> Server { deferring_in_chunks(body, 256 * 1024, Duration::ZERO) @@ -298,11 +156,6 @@ fn rejected(error: ClientError) -> (SocketAddr, String, String) { } } -/// A scripted server whose every answer is `reply`. -fn answering(reply: CultNetMessage) -> Server { - scripted(move |_| reply.clone()) -} - fn nothing_found(message_id: &str, operation: &str) -> CultNetMessage { encode_response(message_id, operation, &HuginnMindResponse::View(None), "scripted").unwrap() } diff --git a/crates/eureka-state/tests/common/mod.rs b/crates/eureka-state/tests/common/mod.rs new file mode 100644 index 0000000..5729e68 --- /dev/null +++ b/crates/eureka-state/tests/common/mod.rs @@ -0,0 +1,174 @@ +//! The daemon half of the fixtures: an in-process Huginn daemon over a +//! temporary mind, on a loopback port, shared by the client and the server +//! tests. +#![allow(dead_code)] + +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::thread::JoinHandle; +use std::time::Duration; + +use chrono::Utc; +use cultnet_rs::{CultNetMessage, CultNetRudpServerEvent, CultNetWireContract, decode_cultnet_message_from_slice}; +use huginn_daemon::serve::{bind, run, schema_registry}; +use huginn_daemon::{Daemon, Handled, Hits, IndexSink, SearchTicket, ServeOptions}; +use huginn_mind::epiphany_pipeline::{ + Date, DocRef, OrgRepo, PipelineCampaign, PipelineDocument, PipelineInstance, Sha, Short, Slug, Title, +}; +use huginn_mind::{ + Faculty, HuginnMindRequest, HuginnMindResponse, IndexStatus, Mind, MindStore, OwnedRedbMessagePackBackingStore, + PipelineAdmissionBatch, PipelineAdmissionOutcome, PipelineProvenance, +}; +use tempfile::TempDir; + +pub const INSTANCE: &str = "eureka"; + +pub fn slug(value: &str) -> Slug { + Slug(value.into()) +} + +pub fn identity(name: &str) -> PipelineDocument { + PipelineDocument::Instance(PipelineInstance { + instance: slug(name), + display_name: Short(format!("{name} mind")), + created_at: Date("2026-09-29".into()), + host: Short(name.into()), + }) +} + +pub fn batch(instance: &str, documents: Vec) -> PipelineAdmissionBatch { + PipelineAdmissionBatch { + instance: slug(instance), + provenance: PipelineProvenance { + faculty: Faculty::Hands, + agent: Short("claude".into()), + session: Short("session-1".into()), + tool: Short("admit".into()), + }, + documents, + } +} + +/// No index: the client's subject is the transport, not the projection. +pub struct Inert; + +impl IndexSink for Inert { + fn committed(&mut self, _mind: &Mind, _writes: &[PipelineRef]) -> anyhow::Result<()> { + Ok(()) + } + fn status(&self) -> IndexStatus { + IndexStatus::Current + } + fn search(&mut self, _text: &str, _top_k: u32) -> Result { + Err("this index cannot search".into()) + } + fn search_deadline(&self) -> Duration { + Duration::ZERO + } + fn searched(&mut self) -> Vec<(SearchTicket, Result)> { + Vec::new() + } + fn abandon(&mut self, _ticket: SearchTicket) {} +} + +pub type TestDaemon = Daemon; + +pub fn answered(handled: Handled) -> HuginnMindResponse { + match handled { + Handled::Answered(response) => response, + Handled::Searching(_) => panic!("the inert index cannot search"), + } +} + +/// A daemon over a temporary mind whose identity is admitted, and whatever +/// else `seed` admits. +pub fn mind(seed: Vec>) -> (TempDir, TestDaemon) { + let root = tempfile::tempdir().unwrap(); + let mut daemon = Daemon::new(Mind::open(root.path(), &slug(INSTANCE)).unwrap(), Inert); + for documents in std::iter::once(vec![identity(INSTANCE)]).chain(seed) { + let outcome = answered(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, documents)), Utc::now())); + assert!(matches!(outcome, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. })), "{outcome:?}"); + } + (root, daemon) +} + +/// A server on a loopback port that stops when dropped. +pub struct Server { + pub addr: SocketAddr, + pub stopping: Arc, + pub thread: Option>, + /// The sessions a scripted server has seen end: what the client's + /// disconnect looks like from the daemon's side. + pub ended: Arc, + pub _root: Option, +} + +impl Drop for Server { + fn drop(&mut self) { + self.stopping.store(true, Ordering::Relaxed); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +pub fn serve(root: TempDir, mut daemon: TestDaemon) -> Server { + let mut hub = bind("127.0.0.1:0".parse().unwrap(), &daemon.runtime_id()).unwrap(); + let addr = hub.local_addr().unwrap(); + let stopping = Arc::new(AtomicBool::new(false)); + let loop_stopping = Arc::clone(&stopping); + let registry = schema_registry().unwrap(); + let thread = std::thread::spawn(move || { + run(&mut daemon, &mut hub, ®istry, &loop_stopping, &ServeOptions::default()).unwrap(); + }); + Server { addr, stopping, thread: Some(thread), ended: Arc::default(), _root: Some(root) } +} + +/// A server that answers each schema message with whatever `script` says. +pub fn scripted(mut script: impl FnMut(&CultNetMessage) -> CultNetMessage + Send + 'static) -> Server { + let mut hub = bind("127.0.0.1:0".parse().unwrap(), "scripted").unwrap(); + let addr = hub.local_addr().unwrap(); + let stopping = Arc::new(AtomicBool::new(false)); + let loop_stopping = Arc::clone(&stopping); + let ended = Arc::new(AtomicUsize::new(0)); + let loop_ended = Arc::clone(&ended); + let thread = std::thread::spawn(move || { + while !loop_stopping.load(Ordering::Relaxed) { + hub.poll_resends().unwrap(); + while let Some(event) = hub.receive_event_once().unwrap() { + if matches!(event, CultNetRudpServerEvent::Disconnected { .. }) { + loop_ended.fetch_add(1, Ordering::Relaxed); + } + let CultNetRudpServerEvent::Frame { session, frame } = event else { continue }; + let message = + decode_cultnet_message_from_slice(&frame.payload, CultNetWireContract::CultNetSchemaV0).unwrap(); + hub.send_schema_message(&session, &script(&message)).unwrap(); + } + std::thread::sleep(Duration::from_millis(2)); + } + }); + Server { addr, stopping, thread: Some(thread), ended, _root: None } +} + +/// A scripted server whose every answer is `reply`. +pub fn answering(reply: CultNetMessage) -> Server { + scripted(move |_| reply.clone()) +} + +pub const CAMPAIGN: &str = "eureka-state"; +pub const REPO: &str = "GameCult/Epiphany"; + +pub fn sha() -> Sha { + Sha("5f98228d".into()) +} + +pub fn campaign() -> PipelineDocument { + PipelineDocument::Campaign(PipelineCampaign { + slug: slug(CAMPAIGN), + title: Title("Eureka pipeline state".into()), + repos: vec![OrgRepo(REPO.into())], + working_branch: Short("hands/cut13a".into()), + target_doc: DocRef { path: Short("notes/target.md".into()), start_line: 1, end_line: 9, commit: sha() }, + }) +} From e0f07c3f53cbf9496bf12b3235832a642a27c3ef Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:36:33 +0200 Subject: [PATCH 031/520] eureka-state: the MCP server, four tools 1:1 with the wire The eureka-state binary serves whoami, admit, view and query over stdio (rmcp 2.2.0), each one HuginnClient call. EUREKA_INSTANCE and HUGINN_ENDPOINT are process configuration: the server fills in the instance and provenance.tool, so input can name neither. Bad configuration never stops startup; whoami reports it and every other tool refuses with the typed body. A mind's refusal is a normal result; Unavailable, Rejected and Misconfigured are results with isError. The query input schema bundles CultLib's selection schema into $defs (vendored; a test pins it to the cultnet-rs revision). Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 254 ++++++++++++ crates/eureka-state/Cargo.toml | 15 +- .../schemas/cultnet.selection.schema.json | 100 +++++ crates/eureka-state/src/main.rs | 260 +++++++++++++ crates/eureka-state/tests/mcp.rs | 366 ++++++++++++++++++ 5 files changed, 992 insertions(+), 3 deletions(-) create mode 100644 crates/eureka-state/schemas/cultnet.selection.schema.json create mode 100644 crates/eureka-state/src/main.rs create mode 100644 crates/eureka-state/tests/mcp.rs diff --git a/Cargo.lock b/Cargo.lock index f278655..71de095 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,17 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -258,6 +269,40 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + [[package]] name = "der" version = "0.7.10" @@ -341,8 +386,13 @@ dependencies = [ "cultnet-rs", "huginn-daemon", "huginn-mind", + "rmcp", "rmp-serde", + "schemars", + "serde", + "serde_json", "tempfile", + "tokio", ] [[package]] @@ -373,12 +423,71 @@ dependencies = [ "winapi", ] +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + [[package]] name = "futures-task" version = "0.3.34" @@ -391,8 +500,13 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -536,6 +650,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "inout" version = "0.1.4" @@ -607,6 +727,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -747,6 +873,40 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "rmcp" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14db48ee17a9ba61810ab1a9c1beb7d06d8136ae39ac25a1137f10d357af01af" +dependencies = [ + "async-trait", + "chrono", + "futures", + "pastey", + "pin-project-lite", + "rmcp-macros", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "rmcp-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "783d787bf21813b285f13019adc49e11af501c658890c1e519f31f937c68b7e3" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "serde_json", + "syn 2.0.119", +] + [[package]] name = "rmp" version = "0.8.15" @@ -811,6 +971,7 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ + "chrono", "dyn-clone", "ref-cast", "schemars_derive", @@ -972,6 +1133,12 @@ dependencies = [ "der", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -1013,6 +1180,93 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.20.1" diff --git a/crates/eureka-state/Cargo.toml b/crates/eureka-state/Cargo.toml index 0de0909..be76e34 100644 --- a/crates/eureka-state/Cargo.toml +++ b/crates/eureka-state/Cargo.toml @@ -5,18 +5,27 @@ edition.workspace = true license.workspace = true publish.workspace = true -# The client core: one call over CultNet to a Huginn daemon. It owns the -# transport and nothing of the mind. The operation envelope, the wire +# The client core (`lib.rs`): one call over CultNet to a Huginn daemon. It owns +# the transport and nothing of the mind. The operation envelope, the wire # vocabulary and the deferred-body bound are `huginn-mind`'s, so the client and # the daemon cannot spell them differently; the content fetch and its # verification are `cultnet-rs`'s. +# +# The MCP server (`main.rs`, the `eureka-state` binary) is the client's one +# consumer: four tools over stdio, one per wire operation, holding no state. [dependencies] anyhow = "1" cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "6251cbcd37c12ef3f0c1cc50063a03369225b7ac", package = "cultnet-rs" } huginn-mind = { path = "../huginn-mind" } +rmcp = { version = "2.2.0", default-features = false, features = ["server", "macros", "transport-io"] } rmp-serde = "1" +schemars = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tokio = { version = "1", features = ["rt", "macros", "io-std"] } -# Tests run a real daemon in-process against a temporary mind. +# Tests run a real daemon in-process against a temporary mind, and the real +# binary against it. [dev-dependencies] chrono = "0.4.44" huginn-daemon = { path = "../huginn-daemon" } diff --git a/crates/eureka-state/schemas/cultnet.selection.schema.json b/crates/eureka-state/schemas/cultnet.selection.schema.json new file mode 100644 index 0000000..7076428 --- /dev/null +++ b/crates/eureka-state/schemas/cultnet.selection.schema.json @@ -0,0 +1,100 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/GameCult/CultLib/contracts/cultnet/cultnet.selection.schema.json", + "title": "CultNet Typed Selection", + "type": "object", + "additionalProperties": false, + "properties": { + "schemas": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "minItems": 1, + "description": "Absent means every schema. Present and empty is refused at the door (selection_invalid), never treated as \"every schema\": omit the field instead." + }, + "keys": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "minItems": 1, + "description": "Absent means every key. Present and empty is refused at the door (selection_invalid), never treated as \"every key\": omit the field instead." + }, + "fields": { + "type": "array", + "items": { "$ref": "#/$defs/fieldPredicate" } + }, + "cites": { "$ref": "#/$defs/citation" }, + "cited": { "$ref": "#/$defs/incoming" }, + "projection": { + "type": "string", + "enum": ["header", "document"], + "default": "header" + }, + "descending": { "type": "boolean", "default": false }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 200, + "description": "Clamped to 1..=200 by the answering server, not refused outside that range: an absent value or one outside the bound is treated as the nearer bound (absent = 200)." + }, + "cursor": { "type": "string" } + }, + "$defs": { + "recordRef": { + "type": "object", + "required": ["schemaId", "recordKey"], + "additionalProperties": false, + "properties": { + "schemaId": { "type": "string", "minLength": 1 }, + "recordKey": { "type": "string", "minLength": 1 } + } + }, + "fieldPredicate": { + "type": "object", + "required": ["index", "op"], + "additionalProperties": false, + "properties": { + "index": { "type": "string", "minLength": 1 }, + "op": { "type": "string", "enum": ["any_of", "lt", "le", "ge", "gt"] }, + "values": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "number": { + "type": "string", + "description": "Q-J: a canonical decimal string, never a JSON number. Matches ^-?(0|[1-9][0-9]*)(\\.[0-9]*[1-9])?$ and is not \"-0\"; any other spelling is refused typed at the door (selection_invalid), never normalised.", + "pattern": "^(?!-0$)-?(0|[1-9][0-9]*)(\\.[0-9]*[1-9])?$" + } + }, + "oneOf": [ + { + "required": ["values"], + "not": { "required": ["number"] }, + "properties": { "op": { "const": "any_of" } } + }, + { + "required": ["number"], + "not": { "required": ["values"] }, + "properties": { "op": { "enum": ["lt", "le", "ge", "gt"] } } + } + ] + }, + "citation": { + "type": "object", + "required": ["target"], + "additionalProperties": false, + "properties": { + "target": { "$ref": "#/$defs/recordRef" }, + "role": { "type": "string", "minLength": 1 } + } + }, + "incoming": { + "type": "object", + "required": ["role", "exists"], + "additionalProperties": false, + "properties": { + "role": { "type": "string", "minLength": 1 }, + "exists": { "type": "boolean" } + } + } + } +} diff --git a/crates/eureka-state/src/main.rs b/crates/eureka-state/src/main.rs new file mode 100644 index 0000000..a7f4a58 --- /dev/null +++ b/crates/eureka-state/src/main.rs @@ -0,0 +1,260 @@ +//! `eureka-state`: Huginn's mind as four MCP tools over stdio. +//! +//! One tool per wire operation (`whoami`, `admit`, `view`, `query`), each a +//! `HuginnClient` call and nothing else. The declared identity is this +//! process's configuration, not a per-call argument: `EUREKA_INSTANCE` names +//! the mind and `HUGINN_ENDPOINT` (`rudp://host:port`) the daemon that holds +//! it. The server fills in `instance` and `provenance.tool`, so a caller can +//! name neither. +//! +//! Bad configuration does not stop the server: an MCP server that dies at +//! startup vanishes from Claude Code without a word. `whoami` reports it and +//! every other tool refuses with it. A refusal from the mind is an answer and +//! comes back as a normal result; a daemon that cannot be reached or that +//! rejects the envelope is a result with `isError: true`, typed, so a caller +//! can tell a down daemon from a permanent mismatch. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use cultnet_rs::Selection; +use eureka_state::{ClientError, HuginnClient}; +use huginn_mind::epiphany_pipeline::{PipelineDocument, PipelineRef, Short, Slug}; +use huginn_mind::{ + Faculty, HuginnMindRequest, HuginnMindResponse, PipelineAdmissionBatch, PipelineProvenance, SemanticQuery, +}; +use rmcp::handler::server::router::tool::ToolRouter; +use rmcp::handler::server::wrapper::Parameters; +use rmcp::model::{CallToolResult, ContentBlock, ServerCapabilities, ServerInfo}; +use rmcp::{ServerHandler, ServiceExt, tool, tool_handler, tool_router}; +use schemars::{JsonSchema, Schema, SchemaGenerator}; +use serde::Deserialize; +use serde_json::{Value, json}; + +/// The `provenance.tool` every admission from this server carries. +const TOOL: &str = "eureka-state"; +/// How long one call may take, connection to last byte. +const TIMEOUT: Duration = Duration::from_secs(15); + +/// CultLib's published shape of a selection, vendored because the selection +/// type carries no schema of its own. `selection_schema_matches_cultlib` pins +/// it to the CultLib revision `cultnet-rs` is built from. +const SELECTION_SCHEMA: &str = include_str!("../schemas/cultnet.selection.schema.json"); + +/// The selection's schema, self-contained: its own `$defs` move into the +/// tool's, so every `$ref` in the input schema is `#/$defs/...`. +fn selection_schema(generator: &mut SchemaGenerator) -> Schema { + let Ok(Value::Object(mut selection)) = serde_json::from_str(SELECTION_SCHEMA) else { + unreachable!("the vendored selection schema is a JSON object"); + }; + selection.remove("$schema"); + selection.remove("$id"); + if let Some(Value::Object(defs)) = selection.remove("$defs") { + generator.definitions_mut().extend(defs); + } + Schema::try_from(Value::Object(selection)).expect("an object is a schema") +} + +#[derive(Deserialize, JsonSchema)] +struct AdmitInput { + /// The faculty the batch is admitted as. Attribution, not authority. + faculty: Faculty, + /// The admitting agent's name. + agent: Short, + /// The admitting session, recorded on the receipt so that several sessions + /// sharing one mind stay tellable apart. + session: Short, + /// The documents to admit as one batch. The mind admits them whole or refuses them. + documents: Vec, +} + +#[derive(Deserialize, JsonSchema)] +struct ViewInput { + /// The document's id, as a query or an admission returned it. + id: PipelineRef, +} + +#[derive(Deserialize, JsonSchema)] +struct QueryInput { + /// A CultNet typed selection over the mind's documents. + #[schemars(schema_with = "selection_schema")] + selection: Selection, + /// Optional: rank by nearness to this text instead of ordering by key. + semantic: Option, +} + +/// Why a call produced no answer. Each is a typed body with `isError: true`. +enum Trouble { + Misconfigured(String), + Client(ClientError), + Internal(String), +} + +impl Trouble { + fn body(&self) -> Value { + match self { + Self::Misconfigured(detail) => json!({ "error": "Misconfigured", "detail": detail }), + Self::Client(ClientError::Unavailable { endpoint, detail }) => { + json!({ "error": "Unavailable", "endpoint": format!("rudp://{endpoint}"), "detail": detail }) + } + Self::Client(ClientError::Rejected { endpoint, code, detail }) => { + json!({ "error": "Rejected", "endpoint": format!("rudp://{endpoint}"), "code": code, "detail": detail }) + } + Self::Internal(detail) => json!({ "error": "Internal", "detail": detail }), + } + } +} + +/// The two environment values as given, and the client they make when both are +/// valid. +struct Settings { + instance: Option, + endpoint: Option, + client: Result, +} + +impl Settings { + fn from_env() -> Self { + let instance = std::env::var("EUREKA_INSTANCE").ok(); + let endpoint = std::env::var("HUGINN_ENDPOINT").ok(); + let client = client(instance.as_deref(), endpoint.as_deref()); + Self { instance, endpoint, client } + } +} + +fn client(instance: Option<&str>, endpoint: Option<&str>) -> Result { + let instance = instance.ok_or("EUREKA_INSTANCE is not set")?; + let slug = Slug(instance.to_string()); + slug.validate_slug().map_err(|refusal| format!("EUREKA_INSTANCE is not a valid instance name: {refusal:?}"))?; + let endpoint = endpoint.ok_or("HUGINN_ENDPOINT is not set")?; + let address = endpoint + .strip_prefix("rudp://") + .and_then(|address| address.parse::().ok()) + .ok_or_else(|| format!("HUGINN_ENDPOINT is not rudp://:: {endpoint}"))?; + Ok(HuginnClient::new(address, slug, TIMEOUT)) +} + +#[derive(Clone)] +struct EurekaState { + settings: Arc, + tool_router: ToolRouter, +} + +impl EurekaState { + fn new(settings: Settings) -> Self { + Self { settings: Arc::new(settings), tool_router: Self::tool_router() } + } + + /// One wire call for the configured instance. The request is built from + /// that instance, never from anything the caller sent. + async fn call(&self, request: impl FnOnce(Slug) -> HuginnMindRequest) -> Result { + let client = self.settings.client.as_ref().map_err(|detail| Trouble::Misconfigured(detail.clone()))?.clone(); + let request = request(client.instance().clone()); + tokio::task::spawn_blocking(move || client.call(request)) + .await + .map_err(|error| Trouble::Internal(error.to_string()))? + .map_err(Trouble::Client) + } + + async fn answer(&self, request: impl FnOnce(Slug) -> HuginnMindRequest) -> CallToolResult { + match self.call(request).await { + Ok(response) => reply(payload(response), false), + Err(trouble) => reply(trouble.body(), true), + } + } +} + +/// The answer as the tool's output: the operation's own payload, or the typed +/// refusal, tagged, when the mind refused the request. +fn payload(response: HuginnMindResponse) -> Value { + let value = match response { + HuginnMindResponse::Whoami(status) => serde_json::to_value(status), + HuginnMindResponse::Admit(outcome) => serde_json::to_value(outcome), + HuginnMindResponse::View(view) => serde_json::to_value(view), + HuginnMindResponse::Query(page) => serde_json::to_value(page), + other => serde_json::to_value(other), + }; + value.expect("wire types serialize") +} + +fn reply(body: Value, is_error: bool) -> CallToolResult { + let content = vec![ContentBlock::text(body.to_string())]; + if is_error { CallToolResult::error(content) } else { CallToolResult::success(content) } +} + +#[tool_router] +impl EurekaState { + #[tool( + description = "Who this server acts for and whether the Huginn daemon answers: the configured instance and endpoint, reachable, and the mind's status when it does. Reports bad configuration instead of failing." + )] + async fn whoami(&self) -> CallToolResult { + let mut out = json!({ + "instance": self.settings.instance, + "endpoint": self.settings.endpoint, + "reachable": false, + "status": null, + }); + match self.call(|_| HuginnMindRequest::Whoami).await { + Ok(HuginnMindResponse::Whoami(status)) => { + out["reachable"] = json!(true); + out["status"] = serde_json::to_value(status).expect("wire types serialize"); + } + Ok(other) => out["error"] = json!({ "error": "Internal", "detail": format!("unexpected answer: {other:?}") }), + Err(trouble) => out["error"] = trouble.body(), + } + reply(out, false) + } + + #[tool( + description = "Admit one batch of pipeline documents into the mind, whole or not at all. Returns the outcome: committed with a receipt, already admitted, conflict, or a typed refusal. The instance is this server's configuration; do not pass one." + )] + async fn admit(&self, Parameters(input): Parameters) -> CallToolResult { + self.answer(|instance| { + HuginnMindRequest::Admit(PipelineAdmissionBatch { + instance, + provenance: PipelineProvenance { + faculty: input.faculty, + agent: input.agent, + session: input.session, + tool: Short(TOOL.into()), + }, + documents: input.documents, + }) + }) + .await + } + + #[tool(description = "Read one document by id, with its admission facts and status. null when the mind has no such document.")] + async fn view(&self, Parameters(input): Parameters) -> CallToolResult { + self.answer(|instance| HuginnMindRequest::View { instance, id: input.id }).await + } + + #[tool( + description = "Query the mind with a CultNet typed selection, optionally ranked by semantic nearness. Returns one page; pass its cursor back in the selection for the next." + )] + async fn query(&self, Parameters(input): Parameters) -> CallToolResult { + self.answer(|instance| HuginnMindRequest::Query { + instance, + selection: input.selection, + semantic: input.semantic, + }) + .await + } +} + +#[tool_handler] +impl ServerHandler for EurekaState { + fn get_info(&self) -> ServerInfo { + ServerInfo::new(ServerCapabilities::builder().enable_tools().build()).with_instructions( + "Huginn's typed mind for this instance: whoami, admit, view, query. Refusals are answers; isError means the daemon could not be reached or rejected the request.", + ) + } +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + let server = EurekaState::new(Settings::from_env()).serve(rmcp::transport::stdio()).await?; + server.waiting().await?; + Ok(()) +} diff --git a/crates/eureka-state/tests/mcp.rs b/crates/eureka-state/tests/mcp.rs new file mode 100644 index 0000000..adbfcda --- /dev/null +++ b/crates/eureka-state/tests/mcp.rs @@ -0,0 +1,366 @@ +//! The `eureka-state` binary over stdio, as Claude Code drives it: real +//! processes, real JSON-RPC, a real daemon in-process behind it (or a scripted +//! server where the daemon cannot be made to misbehave). + +use std::collections::BTreeSet; +use std::io::{BufRead, BufReader, Write}; +use std::net::UdpSocket; +use std::process::{Child, ChildStdin, Command, Stdio}; +use std::sync::mpsc::{Receiver, channel}; +use std::time::Duration; + +use huginn_mind::envelope::{OperationFailure, encode_failure, encode_response}; +use huginn_mind::HuginnMindResponse; +use huginn_mind::epiphany_pipeline::{PipelineKind, PipelineRef, Short}; +use serde_json::{Value, json}; + +mod common; +use common::*; + +const WAIT: Duration = Duration::from_secs(60); + +struct Mcp { + child: Child, + stdin: ChildStdin, + lines: Receiver, + next: u64, +} + +impl Drop for Mcp { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +impl Mcp { + /// The binary with exactly the environment given, initialised. + fn start(env: &[(&str, &str)]) -> Self { + let mut command = Command::new(env!("CARGO_BIN_EXE_eureka-state")); + command + .env_remove("EUREKA_INSTANCE") + .env_remove("HUGINN_ENDPOINT") + .envs(env.iter().copied()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + let mut child = command.spawn().expect("the binary starts"); + let stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let (sender, lines) = channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + let _ = sender.send(line); + } + }); + let mut mcp = Self { child, stdin, lines, next: 1 }; + let init = mcp.rpc( + "initialize", + json!({ "protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": { "name": "test", "version": "0" } }), + ); + assert!(init.get("result").is_some(), "{init}"); + mcp.send(json!({ "jsonrpc": "2.0", "method": "notifications/initialized" })); + mcp + } + + /// Configured for `instance` against a daemon at `addr`. + fn at(instance: &str, addr: std::net::SocketAddr) -> Self { + Self::start(&[("EUREKA_INSTANCE", instance), ("HUGINN_ENDPOINT", &format!("rudp://{addr}"))]) + } + + fn send(&mut self, message: Value) { + writeln!(self.stdin, "{message}").unwrap(); + self.stdin.flush().unwrap(); + } + + /// One request and its response, whole (`result` or `error`). + fn rpc(&mut self, method: &str, params: Value) -> Value { + let id = self.next; + self.next += 1; + self.send(json!({ "jsonrpc": "2.0", "id": id, "method": method, "params": params })); + loop { + let line = self.lines.recv_timeout(WAIT).expect("the server answers"); + let message: Value = serde_json::from_str(&line).expect("stdout carries only JSON-RPC"); + if message["id"] == json!(id) { + return message; + } + } + } + + /// A tool call's result: whether it is an error, and its JSON body. + fn call(&mut self, tool: &str, arguments: Value) -> (bool, Value) { + let response = self.rpc("tools/call", json!({ "name": tool, "arguments": arguments })); + let result = &response["result"]; + assert!(result.is_object(), "a tool result, not a protocol error: {response}"); + let text = result["content"][0]["text"].as_str().expect("text content"); + (result["isError"] == json!(true), serde_json::from_str(text).expect("a JSON body")) + } +} + +/// A loopback UDP port nothing listens on. +fn closed_port() -> std::net::SocketAddr { + UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap() +} + +fn faculty_args(documents: Vec) -> Value { + json!({ "faculty": "Hands", "agent": "claude", "session": "session-7", "documents": documents }) +} + +fn campaign_json() -> Value { + serde_json::to_value(campaign()).unwrap() +} + +/// Every `$ref` in a schema, and the `$defs` it can resolve against. +fn refs(schema: &Value, found: &mut Vec) { + match schema { + Value::Object(map) => { + for (key, value) in map { + if key == "$ref" { + found.push(value.as_str().unwrap().to_string()); + } else { + refs(value, found); + } + } + } + Value::Array(items) => items.iter().for_each(|item| refs(item, found)), + _ => {} + } +} + +/// Four tools, each input schema a self-contained document: every `$ref` is +/// internal and resolves, and the selection's definitions are bundled in. +#[test] +fn tools_list_offers_four_tools_with_self_contained_schemas() { + let mut mcp = Mcp::at(INSTANCE, closed_port()); + let listed = mcp.rpc("tools/list", json!({})); + let tools = listed["result"]["tools"].as_array().expect("tools"); + let names: BTreeSet<&str> = tools.iter().map(|tool| tool["name"].as_str().unwrap()).collect(); + assert_eq!(names, BTreeSet::from(["whoami", "admit", "view", "query"])); + for tool in tools { + let schema = &tool["inputSchema"]; + let mut found = Vec::new(); + refs(schema, &mut found); + for reference in &found { + let name = reference.strip_prefix("#/$defs/").unwrap_or_else(|| panic!("{}: remote $ref {reference}", tool["name"])); + assert!(schema["$defs"].get(name).is_some(), "{}: {reference} does not resolve", tool["name"]); + } + assert!(schema.get("$id").is_none(), "{}: a remote identity", tool["name"]); + } + let query = tools.iter().find(|tool| tool["name"] == "query").unwrap(); + for bundled in ["fieldPredicate", "citation", "incoming", "recordRef"] { + assert!(query["inputSchema"]["$defs"].get(bundled).is_some(), "{bundled} is not bundled"); + } + assert_eq!(query["inputSchema"]["properties"]["selection"]["properties"]["limit"]["maximum"], json!(200)); +} + +/// The vendored selection schema is CultLib's, at the revision `cultnet-rs` +/// is built from. +#[test] +fn selection_schema_matches_cultlib() { + let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".into()); + let metadata = Command::new(cargo) + .args(["metadata", "--format-version", "1", "--offline", "--locked"]) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .output() + .unwrap(); + assert!(metadata.status.success(), "{}", String::from_utf8_lossy(&metadata.stderr)); + let metadata: Value = serde_json::from_slice(&metadata.stdout).unwrap(); + let manifest = metadata["packages"] + .as_array() + .unwrap() + .iter() + .find(|package| package["name"] == "cultnet-rs") + .expect("cultnet-rs is a dependency")["manifest_path"] + .as_str() + .unwrap(); + let published = std::path::Path::new(manifest).join("../../../contracts/cultnet/cultnet.selection.schema.json"); + let theirs: Value = serde_json::from_str(&std::fs::read_to_string(published).unwrap()).unwrap(); + let ours: Value = serde_json::from_str( + &std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/schemas/cultnet.selection.schema.json")).unwrap(), + ) + .unwrap(); + assert_eq!(ours, theirs); +} + +/// Every tool against a live daemon, and what each returns. +#[test] +fn every_tool_round_trips_against_a_live_daemon() { + let (root, daemon) = mind(vec![]); + let server = serve(root, daemon); + let mut mcp = Mcp::at(INSTANCE, server.addr); + + let (error, whoami) = mcp.call("whoami", json!({})); + assert!(!error); + assert_eq!(whoami["instance"], json!(INSTANCE)); + assert_eq!(whoami["endpoint"], json!(format!("rudp://{}", server.addr))); + assert_eq!(whoami["reachable"], json!(true)); + assert_eq!(whoami["status"]["instance"], json!(INSTANCE)); + assert_eq!(whoami["status"]["documents"], json!(1)); + assert!(whoami.get("error").is_none(), "{whoami}"); + + let (error, admitted) = mcp.call("admit", faculty_args(vec![campaign_json()])); + assert!(!error); + let writes = admitted["Committed"]["writes"].as_array().expect("committed").clone(); + assert_eq!(writes.len(), 1, "{admitted}"); + + let (error, viewed) = mcp.call("view", json!({ "id": writes[0] })); + assert!(!error); + assert_eq!(viewed["id"], writes[0]); + assert_eq!(viewed["admission"]["provenance"]["session"], json!("session-7")); + + let (error, nothing) = mcp.call("view", json!({ "id": absent() })); + assert!(!error); + assert_eq!(nothing, Value::Null); + + let (error, page) = mcp.call("query", json!({ "selection": { "projection": "document" } })); + assert!(!error); + assert_eq!(page["matched"], json!(2), "{page}"); + assert_eq!(page["items"]["Documents"].as_array().unwrap().len(), 2); + + let (_, counted) = mcp.call("whoami", json!({})); + assert_eq!(counted["status"]["receipts"], json!(2)); +} + +/// Declared identity is configuration, not a per-call argument. Input that +/// names another instance, or another tool, changes neither: the batch is +/// admitted for the configured instance, and the receipt says `eureka-state`. +#[test] +fn admit_ignores_any_instance_in_input() { + let (root, daemon) = mind(vec![]); + let server = serve(root, daemon); + let mut mcp = Mcp::at(INSTANCE, server.addr); + + let mut arguments = faculty_args(vec![campaign_json()]); + arguments["instance"] = json!("thought-cage"); + arguments["tool"] = json!("someone-else"); + arguments["provenance"] = json!({ "faculty": "Operator", "agent": "x", "session": "y", "tool": "someone-else" }); + let (error, admitted) = mcp.call("admit", arguments); + assert!(!error); + let writes = admitted["Committed"]["writes"].as_array().unwrap_or_else(|| panic!("not committed: {admitted}")); + + let (_, viewed) = mcp.call("view", json!({ "id": writes[0] })); + let provenance = &viewed["admission"]["provenance"]; + assert_eq!(provenance["tool"], json!("eureka-state")); + assert_eq!(provenance["faculty"], json!("Hands")); + assert_eq!(provenance["agent"], json!("claude")); + assert_eq!(provenance["session"], json!("session-7")); +} + +/// A mind's refusal is an answer: a successful result carrying the typed +/// refusal. Here the configured instance is not the daemon's. +#[test] +fn a_refusal_is_a_successful_result() { + let (root, daemon) = mind(vec![]); + let server = serve(root, daemon); + let mut mcp = Mcp::at("thought-cage", server.addr); + + let (error, viewed) = mcp.call("view", json!({ "id": absent() })); + assert!(!error); + assert!(viewed.get("Refused").is_some(), "{viewed}"); + let (error, queried) = mcp.call("query", json!({ "selection": {} })); + assert!(!error); + assert!(queried.get("Refused").is_some(), "{queried}"); + let (error, admitted) = mcp.call("admit", faculty_args(vec![campaign_json()])); + assert!(!error); + assert!(admitted.get("Refused").is_some(), "{admitted}"); +} + +/// A daemon that cannot be reached is `isError` with the endpoint, and +/// `whoami` reports it as `reachable: false` without a protocol error. +#[test] +fn a_closed_port_is_reported_and_is_not_a_protocol_error() { + let addr = closed_port(); + let mut mcp = Mcp::at(INSTANCE, addr); + + let whoami = mcp.rpc("tools/call", json!({ "name": "whoami", "arguments": {} })); + assert!(whoami.get("error").is_none(), "{whoami}"); + assert_ne!(whoami["result"]["isError"], json!(true)); + let body: Value = serde_json::from_str(whoami["result"]["content"][0]["text"].as_str().unwrap()).unwrap(); + assert_eq!(body["reachable"], json!(false)); + assert_eq!(body["status"], Value::Null); + assert_eq!(body["error"]["error"], json!("Unavailable")); + + let (error, view) = mcp.call("view", json!({ "id": absent() })); + assert!(error); + assert_eq!(view["error"], json!("Unavailable")); + assert_eq!(view["endpoint"], json!(format!("rudp://{addr}"))); +} + +/// A daemon rejecting the envelope is a permanent mismatch, not a down +/// daemon: `isError` with its own code, distinct from `Unavailable`. +#[test] +fn a_rejected_request_is_a_typed_error_distinct_from_unavailable() { + let server = answering(encode_failure( + "eureka-state-call", + "view", + &OperationFailure { code: "wrong-service".into(), message: "not huginn.mind".into() }, + "scripted", + )); + let mut mcp = Mcp::at(INSTANCE, server.addr); + let (error, body) = mcp.call("view", json!({ "id": absent() })); + assert!(error); + assert_eq!(body["error"], json!("Rejected")); + assert_eq!(body["code"], json!("wrong-service")); + assert_eq!(body["detail"], json!("not huginn.mind")); + assert_eq!(body["endpoint"], json!(format!("rudp://{}", server.addr))); +} + +/// A `whoami` answered with another operation's payload is not a mind's +/// status: it is reported, not called reachable. +#[test] +fn whoami_does_not_take_another_answer_for_a_status() { + let server = answering(encode_response("eureka-state-call", "whoami", &HuginnMindResponse::View(None), "scripted").unwrap()); + let mut mcp = Mcp::at(INSTANCE, server.addr); + let (error, body) = mcp.call("whoami", json!({})); + assert!(!error); + assert_eq!(body["reachable"], json!(false)); + assert_eq!(body["status"], Value::Null); + assert!(body["error"]["detail"].as_str().unwrap().contains("unexpected answer"), "{body}"); +} + +/// Bad configuration does not stop the server. `whoami` says what is wrong and +/// every other tool refuses with the same typed body. +#[test] +fn bad_configuration_is_served_and_reported() { + let cases: [(&[(&str, &str)], &str); 5] = [ + (&[("HUGINN_ENDPOINT", "rudp://127.0.0.1:1")], "EUREKA_INSTANCE is not set"), + (&[("EUREKA_INSTANCE", "Not A Slug"), ("HUGINN_ENDPOINT", "rudp://127.0.0.1:1")], "EUREKA_INSTANCE is not a valid"), + (&[("EUREKA_INSTANCE", "eureka")], "HUGINN_ENDPOINT is not set"), + (&[("EUREKA_INSTANCE", "eureka"), ("HUGINN_ENDPOINT", "127.0.0.1:1")], "HUGINN_ENDPOINT is not rudp://"), + (&[("EUREKA_INSTANCE", "eureka"), ("HUGINN_ENDPOINT", "rudp://not-an-address")], "HUGINN_ENDPOINT is not rudp://"), + ]; + for (env, expected) in cases { + let mut mcp = Mcp::start(env); + let (error, whoami) = mcp.call("whoami", json!({})); + assert!(!error, "{expected}"); + assert_eq!(whoami["reachable"], json!(false), "{expected}"); + assert_eq!(whoami["error"]["error"], json!("Misconfigured"), "{expected}"); + assert!(whoami["error"]["detail"].as_str().unwrap().contains(expected), "{expected}: {whoami}"); + for (tool, arguments) in [ + ("view", json!({ "id": absent() })), + ("query", json!({ "selection": {} })), + ("admit", faculty_args(vec![])), + ] { + let (error, refusal) = mcp.call(tool, arguments); + assert!(error, "{tool}: {expected}"); + assert_eq!(refusal["error"], json!("Misconfigured"), "{tool}"); + } + } +} + +/// Input that does not fit the tool's schema is `invalid_params`, a protocol +/// error, and the server keeps serving. +#[test] +fn malformed_input_is_invalid_params() { + let mut mcp = Mcp::at(INSTANCE, closed_port()); + let response = mcp.rpc("tools/call", json!({ "name": "admit", "arguments": { "faculty": "Hands" } })); + assert_eq!(response["error"]["code"], json!(-32602), "{response}"); + let (error, _) = mcp.call("whoami", json!({})); + assert!(!error); +} + +/// A well-formed id of a document no mind here holds. +fn absent() -> Value { + serde_json::to_value(PipelineRef { kind: PipelineKind::CutSpec, id: Short("eureka-state:cut_spec:cut-99.r1".into()) }) + .unwrap() +} From 03eb670db33fa1a1516bdaa05c6bbd1a1c7dd3d7 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:39:03 +0200 Subject: [PATCH 032/520] eureka-state: drop the imports the fixture move left unused, and expect the router field Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/src/main.rs | 1 + crates/eureka-state/tests/client.rs | 23 ++++++++--------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/crates/eureka-state/src/main.rs b/crates/eureka-state/src/main.rs index a7f4a58..e6d529f 100644 --- a/crates/eureka-state/src/main.rs +++ b/crates/eureka-state/src/main.rs @@ -138,6 +138,7 @@ fn client(instance: Option<&str>, endpoint: Option<&str>) -> Result, + #[expect(dead_code, reason = "the tool_handler macro reads this router field")] tool_router: ToolRouter, } diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs index 1ef33e8..bb63d4e 100644 --- a/crates/eureka-state/tests/client.rs +++ b/crates/eureka-state/tests/client.rs @@ -3,30 +3,23 @@ //! made to misbehave. use std::net::{SocketAddr, UdpSocket}; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::thread::JoinHandle; +use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; use chrono::Utc; use cultnet_rs::{ - CultNetMessage, CultNetRudpServerEvent, CultNetWireContract, Selection, answer_content_chunk_request, - decode_cultnet_message_from_slice, encode_cultnet_message_to_vec, pack_content, + CultNetMessage, CultNetWireContract, Selection, answer_content_chunk_request, encode_cultnet_message_to_vec, + pack_content, }; use eureka_state::{ClientError, HuginnClient}; -use huginn_daemon::serve::{MAX_RESPONSE_BYTES, bind, run, schema_registry}; -use huginn_daemon::{Daemon, Handled, Hits, IndexSink, SearchTicket, ServeOptions}; +use huginn_daemon::serve::MAX_RESPONSE_BYTES; use huginn_mind::envelope::{OperationFailure, encode_failure, encode_response}; use huginn_mind::epiphany_pipeline::{ - AuthorityMap, CodeLocation, CutDelete, CutVerification, Date, DocRef, FileChange, Line, NegativeCheck, OrgRepo, - PipelineCampaign, PipelineCutSpec, PipelineDocument, PipelineInstance, PipelineKind, PipelineRef, - PipelineStewardship, Sha, Short, Slug, StructuralDelta, Title, VerificationTest, + AuthorityMap, CodeLocation, CutDelete, CutVerification, Date, FileChange, Line, NegativeCheck, OrgRepo, + PipelineCutSpec, PipelineDocument, PipelineKind, PipelineRef, PipelineStewardship, Short, StructuralDelta, Title, + VerificationTest, }; -use huginn_mind::{ - DeferredAnswer, Faculty, HuginnMindRequest, HuginnMindResponse, IndexStatus, Mind, MindRefusal, MindStore, - OwnedRedbMessagePackBackingStore, PipelineAdmissionBatch, PipelineAdmissionOutcome, PipelineProvenance, -}; -use tempfile::TempDir; +use huginn_mind::{DeferredAnswer, HuginnMindRequest, HuginnMindResponse, MindRefusal, PipelineAdmissionOutcome}; mod common; use common::*; From 295b602b2e86b15113aca0cb8b14cff413d46f25 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 00:42:07 +0200 Subject: [PATCH 033/520] eureka-state tests: common names PipelineRef 8041d82 did not build on its own: the moved Inert index names PipelineRef and the import stayed behind in client.rs. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/tests/common/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/eureka-state/tests/common/mod.rs b/crates/eureka-state/tests/common/mod.rs index 5729e68..e8fc821 100644 --- a/crates/eureka-state/tests/common/mod.rs +++ b/crates/eureka-state/tests/common/mod.rs @@ -14,7 +14,8 @@ use cultnet_rs::{CultNetMessage, CultNetRudpServerEvent, CultNetWireContract, de use huginn_daemon::serve::{bind, run, schema_registry}; use huginn_daemon::{Daemon, Handled, Hits, IndexSink, SearchTicket, ServeOptions}; use huginn_mind::epiphany_pipeline::{ - Date, DocRef, OrgRepo, PipelineCampaign, PipelineDocument, PipelineInstance, Sha, Short, Slug, Title, + Date, DocRef, OrgRepo, PipelineCampaign, PipelineDocument, PipelineInstance, PipelineRef, Sha, Short, Slug, + Title, }; use huginn_mind::{ Faculty, HuginnMindRequest, HuginnMindResponse, IndexStatus, Mind, MindStore, OwnedRedbMessagePackBackingStore, From 63ac5038c96ee591bdec002d89a598d8fba1a006 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 01:01:22 +0200 Subject: [PATCH 034/520] eureka-state tests: seed the stewardship, pin the schema path, malformed input is a tool error Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/tests/client.rs | 14 ++--------- crates/eureka-state/tests/common/mod.rs | 12 +++++++++- crates/eureka-state/tests/mcp.rs | 32 +++++++++++++++---------- 3 files changed, 32 insertions(+), 26 deletions(-) diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs index bb63d4e..56ab80c 100644 --- a/crates/eureka-state/tests/client.rs +++ b/crates/eureka-state/tests/client.rs @@ -15,8 +15,8 @@ use eureka_state::{ClientError, HuginnClient}; use huginn_daemon::serve::MAX_RESPONSE_BYTES; use huginn_mind::envelope::{OperationFailure, encode_failure, encode_response}; use huginn_mind::epiphany_pipeline::{ - AuthorityMap, CodeLocation, CutDelete, CutVerification, Date, FileChange, Line, NegativeCheck, OrgRepo, - PipelineCutSpec, PipelineDocument, PipelineKind, PipelineRef, PipelineStewardship, Short, StructuralDelta, Title, + AuthorityMap, CodeLocation, CutDelete, CutVerification, FileChange, Line, NegativeCheck, OrgRepo, + PipelineCutSpec, PipelineDocument, PipelineKind, PipelineRef, Short, StructuralDelta, Title, VerificationTest, }; use huginn_mind::{DeferredAnswer, HuginnMindRequest, HuginnMindResponse, MindRefusal, PipelineAdmissionOutcome}; @@ -27,16 +27,6 @@ use common::*; const OTHER: &str = "thought-cage"; const TIMEOUT: Duration = Duration::from_secs(20); -fn stewardship() -> PipelineDocument { - PipelineDocument::Stewardship(PipelineStewardship { - instance: slug(INSTANCE), - repo: OrgRepo(REPO.into()), - sequence: 1, - assigned_on: Date("2026-09-29".into()), - note: "assigned".into(), - }) -} - /// A `Line` and a `Short` at the leaf's own bound. fn line(n: usize) -> Line { Line(format!("{n:0>1000}")) diff --git a/crates/eureka-state/tests/common/mod.rs b/crates/eureka-state/tests/common/mod.rs index e8fc821..27a0705 100644 --- a/crates/eureka-state/tests/common/mod.rs +++ b/crates/eureka-state/tests/common/mod.rs @@ -14,7 +14,7 @@ use cultnet_rs::{CultNetMessage, CultNetRudpServerEvent, CultNetWireContract, de use huginn_daemon::serve::{bind, run, schema_registry}; use huginn_daemon::{Daemon, Handled, Hits, IndexSink, SearchTicket, ServeOptions}; use huginn_mind::epiphany_pipeline::{ - Date, DocRef, OrgRepo, PipelineCampaign, PipelineDocument, PipelineInstance, PipelineRef, Sha, Short, Slug, + Date, DocRef, OrgRepo, PipelineCampaign, PipelineDocument, PipelineInstance, PipelineRef, PipelineStewardship, Sha, Short, Slug, Title, }; use huginn_mind::{ @@ -173,3 +173,13 @@ pub fn campaign() -> PipelineDocument { target_doc: DocRef { path: Short("notes/target.md".into()), start_line: 1, end_line: 9, commit: sha() }, }) } + +pub fn stewardship() -> PipelineDocument { + PipelineDocument::Stewardship(PipelineStewardship { + instance: slug(INSTANCE), + repo: OrgRepo(REPO.into()), + sequence: 1, + assigned_on: Date("2026-09-29".into()), + note: "assigned".into(), + }) +} diff --git a/crates/eureka-state/tests/mcp.rs b/crates/eureka-state/tests/mcp.rs index adbfcda..2d9b3ad 100644 --- a/crates/eureka-state/tests/mcp.rs +++ b/crates/eureka-state/tests/mcp.rs @@ -173,7 +173,8 @@ fn selection_schema_matches_cultlib() { .expect("cultnet-rs is a dependency")["manifest_path"] .as_str() .unwrap(); - let published = std::path::Path::new(manifest).join("../../../contracts/cultnet/cultnet.selection.schema.json"); + let crate_dir = std::path::Path::new(manifest).parent().unwrap(); + let published = crate_dir.join("../../contracts/cultnet/cultnet.selection.schema.json"); let theirs: Value = serde_json::from_str(&std::fs::read_to_string(published).unwrap()).unwrap(); let ours: Value = serde_json::from_str( &std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/schemas/cultnet.selection.schema.json")).unwrap(), @@ -185,7 +186,7 @@ fn selection_schema_matches_cultlib() { /// Every tool against a live daemon, and what each returns. #[test] fn every_tool_round_trips_against_a_live_daemon() { - let (root, daemon) = mind(vec![]); + let (root, daemon) = mind(vec![vec![stewardship()]]); let server = serve(root, daemon); let mut mcp = Mcp::at(INSTANCE, server.addr); @@ -195,7 +196,7 @@ fn every_tool_round_trips_against_a_live_daemon() { assert_eq!(whoami["endpoint"], json!(format!("rudp://{}", server.addr))); assert_eq!(whoami["reachable"], json!(true)); assert_eq!(whoami["status"]["instance"], json!(INSTANCE)); - assert_eq!(whoami["status"]["documents"], json!(1)); + assert_eq!(whoami["status"]["documents"], json!(2)); assert!(whoami.get("error").is_none(), "{whoami}"); let (error, admitted) = mcp.call("admit", faculty_args(vec![campaign_json()])); @@ -214,11 +215,11 @@ fn every_tool_round_trips_against_a_live_daemon() { let (error, page) = mcp.call("query", json!({ "selection": { "projection": "document" } })); assert!(!error); - assert_eq!(page["matched"], json!(2), "{page}"); - assert_eq!(page["items"]["Documents"].as_array().unwrap().len(), 2); + assert_eq!(page["matched"], json!(3), "{page}"); + assert_eq!(page["items"]["Documents"].as_array().unwrap().len(), 3); let (_, counted) = mcp.call("whoami", json!({})); - assert_eq!(counted["status"]["receipts"], json!(2)); + assert_eq!(counted["status"]["receipts"], json!(3)); } /// Declared identity is configuration, not a per-call argument. Input that @@ -226,7 +227,7 @@ fn every_tool_round_trips_against_a_live_daemon() { /// admitted for the configured instance, and the receipt says `eureka-state`. #[test] fn admit_ignores_any_instance_in_input() { - let (root, daemon) = mind(vec![]); + let (root, daemon) = mind(vec![vec![stewardship()]]); let server = serve(root, daemon); let mut mcp = Mcp::at(INSTANCE, server.addr); @@ -250,7 +251,7 @@ fn admit_ignores_any_instance_in_input() { /// refusal. Here the configured instance is not the daemon's. #[test] fn a_refusal_is_a_successful_result() { - let (root, daemon) = mind(vec![]); + let (root, daemon) = mind(vec![vec![stewardship()]]); let server = serve(root, daemon); let mut mcp = Mcp::at("thought-cage", server.addr); @@ -348,13 +349,18 @@ fn bad_configuration_is_served_and_reported() { } } -/// Input that does not fit the tool's schema is `invalid_params`, a protocol -/// error, and the server keeps serving. +/// Input that does not fit the tool's schema is refused before any call is +/// made. rmcp 2.2 reports it as a tool result with `isError`, not as a +/// JSON-RPC `invalid_params`, so the model sees what to correct. The server +/// keeps serving. #[test] -fn malformed_input_is_invalid_params() { - let mut mcp = Mcp::at(INSTANCE, closed_port()); +fn malformed_input_is_refused_before_any_call() { + let server = answering(encode_response("eureka-state-call", "admit", &HuginnMindResponse::View(None), "scripted").unwrap()); + let mut mcp = Mcp::at(INSTANCE, server.addr); let response = mcp.rpc("tools/call", json!({ "name": "admit", "arguments": { "faculty": "Hands" } })); - assert_eq!(response["error"]["code"], json!(-32602), "{response}"); + assert!(response.get("error").is_none(), "{response}"); + assert_eq!(response["result"]["isError"], json!(true)); + assert!(response["result"]["content"][0]["text"].as_str().unwrap().contains("missing field"), "{response}"); let (error, _) = mcp.call("whoami", json!({})); assert!(!error); } From 120e0a6e5a0fc2ca14756789c302c0f727084aef Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 01:41:14 +0200 Subject: [PATCH 035/520] eureka-state tests: initialize must advertise the tools capability cargo-mutants on the Cut 13b diff: replacing get_info with Default::default() survived, and a client that does not see a tools capability does not list tools. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/tests/mcp.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/eureka-state/tests/mcp.rs b/crates/eureka-state/tests/mcp.rs index 2d9b3ad..ab5a924 100644 --- a/crates/eureka-state/tests/mcp.rs +++ b/crates/eureka-state/tests/mcp.rs @@ -24,6 +24,8 @@ struct Mcp { stdin: ChildStdin, lines: Receiver, next: u64, + /// The server's answer to `initialize`. + init: Value, } impl Drop for Mcp { @@ -53,12 +55,13 @@ impl Mcp { let _ = sender.send(line); } }); - let mut mcp = Self { child, stdin, lines, next: 1 }; + let mut mcp = Self { child, stdin, lines, next: 1, init: Value::Null }; let init = mcp.rpc( "initialize", json!({ "protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": { "name": "test", "version": "0" } }), ); assert!(init.get("result").is_some(), "{init}"); + mcp.init = init["result"].clone(); mcp.send(json!({ "jsonrpc": "2.0", "method": "notifications/initialized" })); mcp } @@ -370,3 +373,10 @@ fn absent() -> Value { serde_json::to_value(PipelineRef { kind: PipelineKind::CutSpec, id: Short("eureka-state:cut_spec:cut-99.r1".into()) }) .unwrap() } + +/// A client only lists tools from a server that says it has them. +#[test] +fn initialize_advertises_tools() { + let mcp = Mcp::at(INSTANCE, closed_port()); + assert!(mcp.init["capabilities"]["tools"].is_object(), "{}", mcp.init); +} From 1d3f917d9edac5a93595ead910f0982c8361e767 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 02:51:15 +0200 Subject: [PATCH 036/520] huginn-mind: admission validates who admitted the batch A batch's agent and session were committed as given: "", " ", a 5,000 character string and a string with a line break all became the receipt's admitting session. The mind now refuses, at A1 before any document is read, an agent or session that is longer than Short's 200 bytes (FieldBound), blank after trim, or spread over more than one line (InvalidFormat), so every writer gets the check. The refusals are the leaf's own, since the fields are the leaf's Short type; the published wire schema does not change. Short exposes no validator, so the bound is a constant pinned to the maxLength its schema publishes. Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/admission.rs | 87 ++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/crates/huginn-mind/src/admission.rs b/crates/huginn-mind/src/admission.rs index 4b85cdd..0288ad0 100644 --- a/crates/huginn-mind/src/admission.rs +++ b/crates/huginn-mind/src/admission.rs @@ -5,7 +5,8 @@ //! hand-off and import, and Cut 13's tools. The steps run in order and the //! first failing step is the outcome: //! -//! - A1 the batch names this mind; A2 one to sixty-four envelopes; +//! - A1 the batch names this mind, and attributes itself to a named agent and +//! session (`refuse_provenance`); A2 one to sixty-four envelopes; //! - A3 every envelope passes the leaf's bounds, formats and key //! recomputation; A4 identities are unique in the batch; A5 every document //! carrying an instance names this mind; @@ -107,6 +108,7 @@ impl Mind { ) -> Result { // A1 self.require_instance(instance)?; + refuse_provenance(&provenance)?; // A2 if envelopes.is_empty() || envelopes.len() > BATCH_MAX { return Err(MindRefusal::BatchSize { actual: envelopes.len() as u32 }); @@ -171,6 +173,30 @@ impl Mind { } /// A3 through the leaf, then A5. +/// The bound `epiphany_pipeline::Short` declares, in UTF-8 bytes. The leaf +/// exposes no validator for it, so `short_bound_is_the_leafs_own` pins this to +/// the `maxLength` its schema publishes. +const SHORT_MAX: usize = 200; + +/// Who admitted a batch is the receipt's only record of it, so the mind checks +/// it here and every writer, whatever its transport, gets the check: the agent +/// and the session are within `Short`'s bound, hold something besides +/// whitespace, and sit on one line. Refused as the leaf's own refusals, since +/// the fields are the leaf's own type. +fn refuse_provenance(provenance: &PipelineProvenance) -> Result<(), MindRefusal> { + for (field, value) in [("provenance.agent", &provenance.agent), ("provenance.session", &provenance.session)] { + let text = value.0.as_str(); + if text.len() > SHORT_MAX { + return Err(PipelineRefusal::FieldBound { field: field.into(), limit: SHORT_MAX as u32, actual: text.len() as u32 }.into()); + } + let line_break = |c: char| matches!(c, '\n' | '\r' | '\u{0B}' | '\u{0C}' | '\u{85}' | '\u{2028}' | '\u{2029}'); + if text.trim().is_empty() || text.contains(line_break) { + return Err(PipelineRefusal::InvalidFormat { field: field.into(), value: text.into() }.into()); + } + } + Ok(()) +} + fn stage(envelope: CultCacheEnvelope, mind: &Slug) -> Result { validate_pipeline_write_envelope(&envelope)?; let document = PipelineDocument::decode(&envelope)?; @@ -693,6 +719,65 @@ mod tests { assert_eq!(mind.genesis_receipt_id().unwrap(), Some(first)); } + fn admit_attributed(agent: &str, session: &str) -> (Mind, PipelineAdmissionOutcome) { + let mut mind = opened(MemoryStore::new(), INSTANCE); + let provenance = PipelineProvenance { faculty: Faculty::Hands, agent: s(agent), session: s(session), tool: s("admit") }; + let batch = PipelineAdmissionBatch { instance: slug(INSTANCE), provenance, documents: vec![instance(INSTANCE), stewardship(INSTANCE, REPO)] }; + let outcome = mind.admit(batch, now()); + (mind, outcome) + } + + /// Who admitted a batch is validated by the mind, so no writer can commit + /// a blank, multi-line or oversized attribution, and a refused batch leaves + /// nothing behind. + #[test] + fn provenance_names_an_agent_and_a_session_on_one_line_within_short() { + let over = "x".repeat(SHORT_MAX + 1); + let exact = "y".repeat(SHORT_MAX); + for (field, agent, session) in [ + ("provenance.agent", "", "session-1"), + ("provenance.agent", " \t ", "session-1"), + ("provenance.session", "claude", ""), + ("provenance.session", "claude", " "), + ("provenance.session", "claude", "\u{2003}"), + ("provenance.session", "claude", "line\nbreak"), + ("provenance.session", "claude", "line\rbreak"), + ("provenance.session", "claude", "a\u{2028}b"), + ("provenance.agent", "two\nlines", "session-1"), + ] { + let (mind, outcome) = admit_attributed(agent, session); + let refused = refusal(outcome); + assert!( + matches!(&refused, MindRefusal::Document(PipelineRefusal::InvalidFormat { field: named, .. }) if named == field), + "{agent:?}/{session:?}: {refused:?}" + ); + assert!(mind.is_empty(), "{agent:?}/{session:?}: a refused batch wrote"); + } + let (mind, outcome) = admit_attributed(&over, "session-1"); + assert_eq!( + refusal(outcome), + MindRefusal::Document(PipelineRefusal::FieldBound { field: "provenance.agent".into(), limit: 200, actual: 201 }) + ); + assert!(mind.is_empty()); + let (_, outcome) = admit_attributed("claude", &over); + assert_eq!( + refusal(outcome), + MindRefusal::Document(PipelineRefusal::FieldBound { field: "provenance.session".into(), limit: 200, actual: 201 }) + ); + // The bound is inclusive and a name with inner spaces is a name. + let (mind, outcome) = admit_attributed(&exact, "session 7"); + committed(outcome); + let receipts = mind.receipts().unwrap(); + assert_eq!(receipts[0].provenance.agent, s(&exact)); + assert_eq!(receipts[0].provenance.session, s("session 7")); + } + + #[test] + fn short_bound_is_the_leafs_own() { + let schema = serde_json::to_value(schemars::schema_for!(Short)).unwrap(); + assert_eq!(schema["maxLength"], serde_json::json!(SHORT_MAX)); + } + #[test] fn admission_refuses_a_foreign_instance_whatever_the_transport() { let store = MemoryStore::new(); From 890a00b0de2cbc42bc5182a6fc06046497ab5034 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 02:51:50 +0200 Subject: [PATCH 037/520] eureka-state: typed TooLarge, typed InvalidInput, and the pins Soul found missing - The client refuses a request whose encoded size exceeds what one RUDP send carries (65,507 less the 36-byte header and the 6-byte channel id), before connecting, as ClientError::TooLarge { bytes, limit }. It was reported as a retryable Unavailable ("Message too long"). The MCP tools surface it as isError with {"error":"TooLarge","bytes","limit"}. - Arguments that fail to deserialize are still a tool error, now with the same typed body as every other one: {"error":"InvalidInput","detail"}. - A non-UTF-8 EUREKA_INSTANCE or HUGINN_ENDPOINT is reported as not valid UTF-8 and not as unset. - Tests: the schema walk asserts no $id or $schema below any tool's root; the 15 s timeout is measured against a silent daemon; the semantic part of a query is observed on the wire; Internal's and TooLarge's bodies and the InvalidInput rewrite are pinned; attribution is refused through the MCP admit; oversize is TooLarge at and one past the limit. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/src/lib.rs | 31 +++++- crates/eureka-state/src/main.rs | 109 ++++++++++++++++++--- crates/eureka-state/tests/client.rs | 88 ++++++++++++++++- crates/eureka-state/tests/mcp.rs | 147 +++++++++++++++++++++++++++- 4 files changed, 348 insertions(+), 27 deletions(-) diff --git a/crates/eureka-state/src/lib.rs b/crates/eureka-state/src/lib.rs index 94dbfec..1ff2bb7 100644 --- a/crates/eureka-state/src/lib.rs +++ b/crates/eureka-state/src/lib.rs @@ -23,7 +23,7 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow, bail}; use cultnet_rs::{ CULTNET_OPERATION_CONNECTION_ID, CultMesh, CultMeshRudpSocketOptions, CultNetMessage, - CultNetRudpSocketTransportConnection, fetch_content, + CultNetRudpSocketTransportConnection, CultNetWireContract, encode_cultnet_message_to_vec, fetch_content, }; use huginn_mind::envelope::{OperationFailure, decode_response, encode_request}; use huginn_mind::epiphany_pipeline::Slug; @@ -33,15 +33,25 @@ use huginn_mind::{HuginnMindRequest, HuginnMindResponse, MAX_DEFERRED_BODY_BYTES const MESSAGE_ID: &str = "eureka-state-call"; const POLL: Duration = Duration::from_millis(2); +/// The largest encoded message one send carries. The client's RUDP session +/// does not fragment (`max_fragment_bytes` is unset), so a message is one UDP +/// datagram: at most 65,507 bytes over IPv4 (65,535 less the 8-byte UDP and +/// 20-byte IP headers), less RUDP's 36-byte fixed header (`RUDP_FIXED_HEADER_BYTES` +/// in `cultnet-rs`, private there) and the 6-byte `schema` channel id. A larger +/// message makes the operating system refuse the send ("Message too long"). +pub const MAX_REQUEST_BYTES: usize = 65_507 - 36 - "schema".len(); + /// Why a call produced no answer. `Unavailable` is the transport's failure and /// the only one worth retrying: the daemon could not be reached, went quiet /// past the call's deadline, or answered out of protocol. `Rejected` is the -/// daemon refusing the envelope itself, which a retry of the same call cannot -/// change. +/// daemon refusing the envelope itself, and `TooLarge` is the client refusing +/// a request that no single send can carry, before sending it; a retry of the +/// same call cannot change either. #[derive(Debug)] pub enum ClientError { Unavailable { endpoint: SocketAddr, detail: String }, Rejected { endpoint: SocketAddr, code: String, detail: String }, + TooLarge { bytes: usize, limit: usize }, } impl fmt::Display for ClientError { @@ -53,6 +63,9 @@ impl fmt::Display for ClientError { Self::Rejected { endpoint, code, detail } => { write!(f, "the Huginn daemon at rudp://{endpoint} rejected the request: {code}: {detail}") } + Self::TooLarge { bytes, limit } => { + write!(f, "the request encodes to {bytes} bytes and one send carries at most {limit}") + } } } } @@ -96,8 +109,15 @@ impl HuginnClient { let deadline = Instant::now() + self.timeout; let unavailable = |error: anyhow::Error| ClientError::Unavailable { endpoint: self.endpoint, detail: format!("{error:#}") }; + let message = encode_request(MESSAGE_ID, &request, None).map_err(unavailable)?; + let bytes = encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0) + .map_err(unavailable)? + .len(); + if bytes > MAX_REQUEST_BYTES { + return Err(ClientError::TooLarge { bytes, limit: MAX_REQUEST_BYTES }); + } let mut session = self.connect(deadline).map_err(unavailable)?; - let outcome = self.exchange(&mut session, &request, deadline); + let outcome = self.exchange(&mut session, &request, &message, deadline); // Best effort: the daemon drops a session it never hears from, only later. let _ = session.disconnect(Vec::new()); outcome.map_err(|failure| match failure { @@ -114,9 +134,10 @@ impl HuginnClient { &self, session: &mut CultNetRudpSocketTransportConnection, request: &HuginnMindRequest, + message: &CultNetMessage, deadline: Instant, ) -> Result { - let reply = self.ask(session, &encode_request(MESSAGE_ID, request, None)?, deadline)?; + let reply = self.ask(session, message, deadline)?; let CultNetMessage::OperationResponse { operation, .. } = &reply else { return Err(anyhow!("the daemon answered with something other than an operation response").into()); }; diff --git a/crates/eureka-state/src/main.rs b/crates/eureka-state/src/main.rs index e6d529f..d5a8bc5 100644 --- a/crates/eureka-state/src/main.rs +++ b/crates/eureka-state/src/main.rs @@ -11,8 +11,10 @@ //! startup vanishes from Claude Code without a word. `whoami` reports it and //! every other tool refuses with it. A refusal from the mind is an answer and //! comes back as a normal result; a daemon that cannot be reached or that -//! rejects the envelope is a result with `isError: true`, typed, so a caller -//! can tell a down daemon from a permanent mismatch. +//! rejects the envelope, or a request too large for one send, is a result with +//! `isError: true`, typed, so a caller can tell a down daemon from a permanent +//! mismatch. Arguments that do not fit a tool's input are typed the same way +//! (`InvalidInput`), as a tool error and not a protocol one. use std::net::SocketAddr; use std::sync::Arc; @@ -25,9 +27,11 @@ use huginn_mind::{ Faculty, HuginnMindRequest, HuginnMindResponse, PipelineAdmissionBatch, PipelineProvenance, SemanticQuery, }; use rmcp::handler::server::router::tool::ToolRouter; +use rmcp::handler::server::tool::ToolCallContext; use rmcp::handler::server::wrapper::Parameters; -use rmcp::model::{CallToolResult, ContentBlock, ServerCapabilities, ServerInfo}; -use rmcp::{ServerHandler, ServiceExt, tool, tool_handler, tool_router}; +use rmcp::model::{CallToolRequestParams, CallToolResult, ContentBlock, ServerCapabilities, ServerInfo}; +use rmcp::service::RequestContext; +use rmcp::{ErrorData, RoleServer, ServerHandler, ServiceExt, tool, tool_handler, tool_router}; use schemars::{JsonSchema, Schema, SchemaGenerator}; use serde::Deserialize; use serde_json::{Value, json}; @@ -101,6 +105,9 @@ impl Trouble { Self::Client(ClientError::Rejected { endpoint, code, detail }) => { json!({ "error": "Rejected", "endpoint": format!("rudp://{endpoint}"), "code": code, "detail": detail }) } + Self::Client(ClientError::TooLarge { bytes, limit }) => { + json!({ "error": "TooLarge", "bytes": bytes, "limit": limit }) + } Self::Internal(detail) => json!({ "error": "Internal", "detail": detail }), } } @@ -116,18 +123,28 @@ struct Settings { impl Settings { fn from_env() -> Self { - let instance = std::env::var("EUREKA_INSTANCE").ok(); - let endpoint = std::env::var("HUGINN_ENDPOINT").ok(); - let client = client(instance.as_deref(), endpoint.as_deref()); - Self { instance, endpoint, client } + let instance = variable("EUREKA_INSTANCE"); + let endpoint = variable("HUGINN_ENDPOINT"); + let shown = |value: &Result, String>| value.as_ref().ok().cloned().flatten(); + let (shown_instance, shown_endpoint) = (shown(&instance), shown(&endpoint)); + Self { instance: shown_instance, endpoint: shown_endpoint, client: client(instance, endpoint) } } } -fn client(instance: Option<&str>, endpoint: Option<&str>) -> Result { - let instance = instance.ok_or("EUREKA_INSTANCE is not set")?; - let slug = Slug(instance.to_string()); +/// An environment variable: its value, `None` when unset, or why it cannot be read. +fn variable(name: &str) -> Result, String> { + match std::env::var(name) { + Ok(value) => Ok(Some(value)), + Err(std::env::VarError::NotPresent) => Ok(None), + Err(std::env::VarError::NotUnicode(_)) => Err(format!("{name} is not valid UTF-8")), + } +} + +fn client(instance: Result, String>, endpoint: Result, String>) -> Result { + let instance = instance?.ok_or("EUREKA_INSTANCE is not set")?; + let slug = Slug(instance); slug.validate_slug().map_err(|refusal| format!("EUREKA_INSTANCE is not a valid instance name: {refusal:?}"))?; - let endpoint = endpoint.ok_or("HUGINN_ENDPOINT is not set")?; + let endpoint = endpoint?.ok_or("HUGINN_ENDPOINT is not set")?; let address = endpoint .strip_prefix("rudp://") .and_then(|address| address.parse::().ok()) @@ -138,7 +155,6 @@ fn client(instance: Option<&str>, endpoint: Option<&str>) -> Result, - #[expect(dead_code, reason = "the tool_handler macro reads this router field")] tool_router: ToolRouter, } @@ -244,8 +260,32 @@ impl EurekaState { } } +/// Every tool error is a typed JSON body. The ones this server writes already +/// are; the one rmcp writes, for arguments that do not deserialize into the +/// tool's input, is plain text, so it is put into the same shape. +fn typed_error(result: CallToolResult) -> CallToolResult { + if result.is_error != Some(true) { + return result; + } + let Some(text) = result.content.first().and_then(ContentBlock::as_text).map(|text| text.text.as_str()) else { + return result; + }; + if serde_json::from_str::(text).is_ok_and(|body| body.is_object()) { + return result; + } + reply(json!({ "error": "InvalidInput", "detail": text }), true) +} + #[tool_handler] impl ServerHandler for EurekaState { + async fn call_tool( + &self, + request: CallToolRequestParams, + context: RequestContext, + ) -> Result { + Ok(typed_error(self.tool_router.call(ToolCallContext::new(self, request, context)).await?)) + } + fn get_info(&self) -> ServerInfo { ServerInfo::new(ServerCapabilities::builder().enable_tools().build()).with_instructions( "Huginn's typed mind for this instance: whoami, admit, view, query. Refusals are answers; isError means the daemon could not be reached or rejected the request.", @@ -259,3 +299,46 @@ async fn main() -> anyhow::Result<()> { server.waiting().await?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn body_of(result: &CallToolResult) -> (bool, String) { + (result.is_error == Some(true), result.content[0].as_text().unwrap().text.clone()) + } + + /// Reaching `Internal` for real takes a panic in a blocking task, so its + /// body is pinned where it is built. + #[test] + fn every_trouble_has_its_typed_body() { + assert_eq!(Trouble::Internal("task panicked".into()).body(), json!({ "error": "Internal", "detail": "task panicked" })); + assert_eq!( + Trouble::Client(ClientError::TooLarge { bytes: 70_000, limit: 65_000 }).body(), + json!({ "error": "TooLarge", "bytes": 70_000, "limit": 65_000 }) + ); + assert_eq!(Trouble::Misconfigured("bad".into()).body(), json!({ "error": "Misconfigured", "detail": "bad" })); + } + + /// Only the text rmcp writes is rewritten: our own typed bodies and every + /// success pass through untouched. + #[test] + fn typed_error_rewrites_only_rmcp_plain_text() { + let plain = CallToolResult::error(vec![ContentBlock::text("failed to deserialize parameters: missing field `faculty`")]); + let (is_error, text) = body_of(&typed_error(plain)); + assert!(is_error); + assert_eq!( + serde_json::from_str::(&text).unwrap(), + json!({ "error": "InvalidInput", "detail": "failed to deserialize parameters: missing field `faculty`" }) + ); + + let typed = reply(json!({ "error": "Unavailable", "detail": "down" }), true); + assert_eq!(body_of(&typed_error(typed.clone())), body_of(&typed)); + + let success = CallToolResult::success(vec![ContentBlock::text("not json, and fine")]); + assert_eq!(body_of(&typed_error(success)), (false, "not json, and fine".to_string())); + + let quoted = CallToolResult::error(vec![ContentBlock::text("[1, 2]")]); + assert_eq!(serde_json::from_str::(&body_of(&typed_error(quoted)).1).unwrap()["error"], json!("InvalidInput")); + } +} diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs index 56ab80c..38cfe14 100644 --- a/crates/eureka-state/tests/client.rs +++ b/crates/eureka-state/tests/client.rs @@ -11,9 +11,9 @@ use cultnet_rs::{ CultNetMessage, CultNetWireContract, Selection, answer_content_chunk_request, encode_cultnet_message_to_vec, pack_content, }; -use eureka_state::{ClientError, HuginnClient}; +use eureka_state::{ClientError, HuginnClient, MAX_REQUEST_BYTES}; use huginn_daemon::serve::MAX_RESPONSE_BYTES; -use huginn_mind::envelope::{OperationFailure, encode_failure, encode_response}; +use huginn_mind::envelope::{OperationFailure, encode_failure, encode_request, encode_response}; use huginn_mind::epiphany_pipeline::{ AuthorityMap, CodeLocation, CutDelete, CutVerification, FileChange, Line, NegativeCheck, OrgRepo, PipelineCutSpec, PipelineDocument, PipelineKind, PipelineRef, Short, StructuralDelta, Title, @@ -128,14 +128,14 @@ fn client(addr: SocketAddr) -> HuginnClient { fn unavailable(error: ClientError) -> (SocketAddr, String) { match error { ClientError::Unavailable { endpoint, detail } => (endpoint, detail), - rejected @ ClientError::Rejected { .. } => panic!("expected Unavailable, got {rejected}"), + other @ (ClientError::Rejected { .. } | ClientError::TooLarge { .. }) => panic!("expected Unavailable, got {other}"), } } fn rejected(error: ClientError) -> (SocketAddr, String, String) { match error { ClientError::Rejected { endpoint, code, detail } => (endpoint, code, detail), - unavailable @ ClientError::Unavailable { .. } => panic!("expected Rejected, got {unavailable}"), + other @ (ClientError::Unavailable { .. } | ClientError::TooLarge { .. }) => panic!("expected Rejected, got {other}"), } } @@ -380,3 +380,83 @@ fn a_body_over_the_deferral_cap_is_refused() { let (_, detail) = unavailable(client(server.addr).call(HuginnMindRequest::Whoami).unwrap_err()); assert!(detail.contains("exceeds"), "{detail}"); } + +/// An admission whose attribution is `agent_pad` and `session_pad` characters +/// long: the two knobs that size a request without changing what it asks. +fn padded_admit(agent_pad: usize, session_pad: usize) -> HuginnMindRequest { + let mut request = batch(INSTANCE, vec![stewardship()]); + request.provenance.agent = Short("a".repeat(agent_pad)); + request.provenance.session = Short("s".repeat(session_pad)); + HuginnMindRequest::Admit(request) +} + +/// What one send has to carry for `request`, measured the way the client does. +fn encoded_bytes(request: &HuginnMindRequest) -> usize { + let message = encode_request("eureka-state-call", request, None).unwrap(); + encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).unwrap().len() +} + +/// A request that encodes to exactly `target` bytes: the session padding is +/// searched for each agent padding until the base64 steps land on it. +fn request_of_size(target: usize) -> HuginnMindRequest { + for agent_pad in 1..16 { + let (mut low, mut high) = (0, target); + while low < high { + let middle = (low + high) / 2; + if encoded_bytes(&padded_admit(agent_pad, middle)) < target { + low = middle + 1; + } else { + high = middle; + } + } + let request = padded_admit(agent_pad, low); + if encoded_bytes(&request) == target { + return request; + } + } + panic!("no padding encodes to {target} bytes"); +} + +fn admitted_answer() -> CultNetMessage { + encode_response( + "eureka-state-call", + "admit", + &HuginnMindResponse::Admit(PipelineAdmissionOutcome::AlreadyAdmitted { receipt_id: "r".into() }), + "scripted", + ) + .unwrap() +} + +/// A request larger than one send carries is refused by the client before it +/// connects to anything, as its own error and not as a transport failure the +/// caller would retry; one byte under the limit is sent, and answered. +#[test] +fn a_request_over_what_one_send_carries_is_too_large_and_one_at_the_limit_is_sent() { + let closed = UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap(); + let over = request_of_size(MAX_REQUEST_BYTES + 1); + let started = Instant::now(); + match client(closed).call(over) { + Err(ClientError::TooLarge { bytes, limit }) => assert_eq!((bytes, limit), (MAX_REQUEST_BYTES + 1, MAX_REQUEST_BYTES)), + other => panic!("expected TooLarge, got {other:?}"), + } + assert!(started.elapsed() < Duration::from_secs(1), "refused only after trying the transport"); + + let server = answering(admitted_answer()); + for size in [MAX_REQUEST_BYTES - 1, MAX_REQUEST_BYTES] { + let answer = client(server.addr).call(request_of_size(size)); + assert!( + matches!(answer, Ok(HuginnMindResponse::Admit(PipelineAdmissionOutcome::AlreadyAdmitted { .. }))), + "a request of {size} bytes: {answer:?}" + ); + } +} + +/// The transport's own refusal of an oversize send ("Message too long") is +/// what an unchecked request used to report, as an `Unavailable` daemon a +/// caller would retry forever. +#[test] +fn an_oversize_admission_is_never_reported_as_an_unavailable_daemon() { + let server = answering(admitted_answer()); + let error = client(server.addr).call(padded_admit(6, 70_000)).unwrap_err(); + assert!(matches!(error, ClientError::TooLarge { .. }), "{error}"); +} diff --git a/crates/eureka-state/tests/mcp.rs b/crates/eureka-state/tests/mcp.rs index ab5a924..f6def4c 100644 --- a/crates/eureka-state/tests/mcp.rs +++ b/crates/eureka-state/tests/mcp.rs @@ -10,7 +10,7 @@ use std::sync::mpsc::{Receiver, channel}; use std::time::Duration; use huginn_mind::envelope::{OperationFailure, encode_failure, encode_response}; -use huginn_mind::HuginnMindResponse; +use huginn_mind::{HuginnMindRequest, HuginnMindResponse}; use huginn_mind::epiphany_pipeline::{PipelineKind, PipelineRef, Short}; use serde_json::{Value, json}; @@ -38,11 +38,17 @@ impl Drop for Mcp { impl Mcp { /// The binary with exactly the environment given, initialised. fn start(env: &[(&str, &str)]) -> Self { + Self::spawn(|command| { + command.envs(env.iter().copied()); + }) + } + + /// The binary with no configuration but what `configure` gives its command. + fn spawn(configure: impl FnOnce(&mut Command)) -> Self { let mut command = Command::new(env!("CARGO_BIN_EXE_eureka-state")); + command.env_remove("EUREKA_INSTANCE").env_remove("HUGINN_ENDPOINT"); + configure(&mut command); command - .env_remove("EUREKA_INSTANCE") - .env_remove("HUGINN_ENDPOINT") - .envs(env.iter().copied()) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()); @@ -130,6 +136,18 @@ fn refs(schema: &Value, found: &mut Vec) { } } +/// Every key named `key` anywhere below `schema`'s root. +fn keys_below_root(schema: &Value, key: &str) -> usize { + fn count(value: &Value, key: &str) -> usize { + match value { + Value::Object(map) => map.iter().map(|(name, inner)| usize::from(name == key) + count(inner, key)).sum(), + Value::Array(items) => items.iter().map(|item| count(item, key)).sum(), + _ => 0, + } + } + schema.as_object().unwrap().values().map(|inner| count(inner, key)).sum() +} + /// Four tools, each input schema a self-contained document: every `$ref` is /// internal and resolves, and the selection's definitions are bundled in. #[test] @@ -147,7 +165,12 @@ fn tools_list_offers_four_tools_with_self_contained_schemas() { let name = reference.strip_prefix("#/$defs/").unwrap_or_else(|| panic!("{}: remote $ref {reference}", tool["name"])); assert!(schema["$defs"].get(name).is_some(), "{}: {reference} does not resolve", tool["name"]); } + // The vendored selection schema brings its own `$id` and `$schema`, + // which would make its `$ref`s resolve against a remote document. assert!(schema.get("$id").is_none(), "{}: a remote identity", tool["name"]); + for key in ["$id", "$schema"] { + assert_eq!(keys_below_root(schema, key), 0, "{}: a nested {key}", tool["name"]); + } } let query = tools.iter().find(|tool| tool["name"] == "query").unwrap(); for bundled in ["fieldPredicate", "citation", "incoming", "recordRef"] { @@ -363,11 +386,125 @@ fn malformed_input_is_refused_before_any_call() { let response = mcp.rpc("tools/call", json!({ "name": "admit", "arguments": { "faculty": "Hands" } })); assert!(response.get("error").is_none(), "{response}"); assert_eq!(response["result"]["isError"], json!(true)); - assert!(response["result"]["content"][0]["text"].as_str().unwrap().contains("missing field"), "{response}"); + let body: Value = serde_json::from_str(response["result"]["content"][0]["text"].as_str().unwrap()).expect("a JSON body"); + assert_eq!(body["error"], json!("InvalidInput"), "{body}"); + assert!(body["detail"].as_str().unwrap().contains("missing field"), "{body}"); let (error, _) = mcp.call("whoami", json!({})); assert!(!error); } +/// A request too large for one send is the client's typed refusal, an error +/// result that names the size and the limit and does not call the daemon +/// unavailable. +#[test] +fn an_oversize_admit_is_a_typed_too_large_error() { + let mut mcp = Mcp::at(INSTANCE, closed_port()); + let mut arguments = faculty_args(vec![]); + arguments["session"] = json!("s".repeat(90_000)); + let (error, body) = mcp.call("admit", arguments); + assert!(error); + assert_eq!(body["error"], json!("TooLarge"), "{body}"); + assert_eq!(body["limit"], json!(eureka_state::MAX_REQUEST_BYTES)); + assert!(body["bytes"].as_u64().unwrap() > eureka_state::MAX_REQUEST_BYTES as u64, "{body}"); +} + +/// Who admitted a batch is validated by the mind, whichever door the batch +/// came through: a blank, whitespace, multi-line or oversized session or agent +/// is a typed refusal, and nothing is committed. +#[test] +fn admit_refuses_an_unusable_attribution() { + let (root, daemon) = mind(vec![]); + let server = serve(root, daemon); + let mut mcp = Mcp::at(INSTANCE, server.addr); + for (field, value) in [ + ("session", "".to_string()), + ("session", " ".to_string()), + ("session", "x".repeat(5_000)), + ("session", "line\nbreak".to_string()), + ("agent", "".to_string()), + ("agent", "two\nlines".to_string()), + ("agent", "y".repeat(201)), + ] { + let mut arguments = faculty_args(vec![campaign_json()]); + arguments[field] = json!(value); + let (error, refused) = mcp.call("admit", arguments); + assert!(!error, "{field}={value:?}"); + let refusal = &refused["Refused"]["Document"]; + let named = json!(format!("provenance.{field}")); + assert!( + refusal["InvalidFormat"]["field"] == named || refusal["FieldBound"]["field"] == named, + "{field}={value:?}: {refused}" + ); + } + let (_, whoami) = mcp.call("whoami", json!({})); + assert_eq!(whoami["status"]["receipts"], json!(1), "only the seed is committed: {whoami}"); + let (error, admitted) = mcp.call("admit", faculty_args(vec![campaign_json()])); + assert!(!error); + assert!(admitted.get("Committed").is_some(), "{admitted}"); +} + +/// The tool forwards the caller's semantic query, and only that, to the mind. +#[test] +fn query_sends_the_semantic_request_on_the_wire() { + let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let sink = std::sync::Arc::clone(&seen); + let server = scripted(move |message| { + let (_, request) = huginn_mind::envelope::decode_request(message).expect("a mind request"); + sink.lock().unwrap().push(request); + encode_response("eureka-state-call", "query", &HuginnMindResponse::View(None), "scripted").unwrap() + }); + let mut mcp = Mcp::at(INSTANCE, server.addr); + mcp.call("query", json!({ "selection": { "projection": "document" }, "semantic": { "text": "drift", "top_k": 7 } })); + mcp.call("query", json!({ "selection": {} })); + let seen = seen.lock().unwrap(); + let [HuginnMindRequest::Query { instance: first, semantic: Some(asked), .. }, HuginnMindRequest::Query { semantic: None, .. }] = + &seen[..] + else { + panic!("two queries expected, the first with its semantic part: {seen:?}"); + }; + assert_eq!(first.0, INSTANCE); + assert_eq!((asked.text.0.as_str(), asked.top_k), ("drift", 7)); +} + +/// A daemon that never answers is given up on after the server's own call +/// timeout, 15 seconds: not earlier, and not much later. +#[test] +fn a_silent_daemon_is_given_up_on_after_fifteen_seconds() { + let silent = UdpSocket::bind("127.0.0.1:0").unwrap(); + let mut mcp = Mcp::at(INSTANCE, silent.local_addr().unwrap()); + let started = std::time::Instant::now(); + let (error, body) = mcp.call("view", json!({ "id": absent() })); + let elapsed = started.elapsed(); + assert!(error); + assert_eq!(body["error"], json!("Unavailable"), "{body}"); + assert!(body["detail"].as_str().unwrap().contains("15s"), "{body}"); + assert!(elapsed >= Duration::from_secs(15), "gave up after {elapsed:?}"); + assert!(elapsed < Duration::from_secs(20), "gave up after {elapsed:?}"); +} + +/// An instance name that is not UTF-8 is reported as that, not as unset. +#[test] +fn a_non_utf8_instance_is_reported_as_such() { + let mut mcp = Mcp::spawn(|command| { + command.env("EUREKA_INSTANCE", non_utf8()).env("HUGINN_ENDPOINT", "rudp://127.0.0.1:1"); + }); + let (error, whoami) = mcp.call("whoami", json!({})); + assert!(!error); + let detail = whoami["error"]["detail"].as_str().unwrap(); + assert!(detail.contains("EUREKA_INSTANCE is not valid UTF-8"), "{whoami}"); + assert_eq!(whoami["instance"], Value::Null); +} + +#[cfg(unix)] +fn non_utf8() -> std::ffi::OsString { + std::os::unix::ffi::OsStringExt::from_vec(vec![b'e', 0xff]) +} + +#[cfg(windows)] +fn non_utf8() -> std::ffi::OsString { + std::os::windows::ffi::OsStringExt::from_wide(&[0x65, 0xd800]) +} + /// A well-formed id of a document no mind here holds. fn absent() -> Value { serde_json::to_value(PipelineRef { kind: PipelineKind::CutSpec, id: Short("eureka-state:cut_spec:cut-99.r1".into()) }) From db40a6647bfd1a691b6c20ee7d3eaba6541dbe48 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 02:54:54 +0200 Subject: [PATCH 038/520] eureka-state tests: size the oversize requests the way base64 allows Request sizes step by four (the payload rides as base64), so a request of exactly the limit is not reachable through the request. The client tests use the largest request that fits and the smallest that does not; the raw-socket test tops a message up with a source runtime id to reach the limit and one byte past it, which pins the limit against the transport's own refusal. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/tests/client.rs | 125 +++++++++++++++++++--------- 1 file changed, 85 insertions(+), 40 deletions(-) diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs index 38cfe14..ff7d50c 100644 --- a/crates/eureka-state/tests/client.rs +++ b/crates/eureka-state/tests/client.rs @@ -8,8 +8,8 @@ use std::time::{Duration, Instant}; use chrono::Utc; use cultnet_rs::{ - CultNetMessage, CultNetWireContract, Selection, answer_content_chunk_request, encode_cultnet_message_to_vec, - pack_content, + CULTNET_OPERATION_CONNECTION_ID, CultMesh, CultMeshRudpSocketOptions, CultNetMessage, CultNetWireContract, Selection, + answer_content_chunk_request, encode_cultnet_message_to_vec, pack_content, }; use eureka_state::{ClientError, HuginnClient, MAX_REQUEST_BYTES}; use huginn_daemon::serve::MAX_RESPONSE_BYTES; @@ -381,40 +381,60 @@ fn a_body_over_the_deferral_cap_is_refused() { assert!(detail.contains("exceeds"), "{detail}"); } -/// An admission whose attribution is `agent_pad` and `session_pad` characters -/// long: the two knobs that size a request without changing what it asks. -fn padded_admit(agent_pad: usize, session_pad: usize) -> HuginnMindRequest { +/// An admission whose session is `session_pad` characters long: the knob that +/// sizes a request without changing what it asks. +fn padded_admit(session_pad: usize) -> HuginnMindRequest { let mut request = batch(INSTANCE, vec![stewardship()]); - request.provenance.agent = Short("a".repeat(agent_pad)); request.provenance.session = Short("s".repeat(session_pad)); HuginnMindRequest::Admit(request) } -/// What one send has to carry for `request`, measured the way the client does. -fn encoded_bytes(request: &HuginnMindRequest) -> usize { - let message = encode_request("eureka-state-call", request, None).unwrap(); - encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0).unwrap().len() +/// What one send has to carry for `message`, measured the way the client does. +fn encoded_bytes(message: &CultNetMessage) -> usize { + encode_cultnet_message_to_vec(message, CultNetWireContract::CultNetSchemaV0).unwrap().len() } -/// A request that encodes to exactly `target` bytes: the session padding is -/// searched for each agent padding until the base64 steps land on it. -fn request_of_size(target: usize) -> HuginnMindRequest { - for agent_pad in 1..16 { - let (mut low, mut high) = (0, target); - while low < high { - let middle = (low + high) / 2; - if encoded_bytes(&padded_admit(agent_pad, middle)) < target { - low = middle + 1; - } else { - high = middle; - } - } - let request = padded_admit(agent_pad, low); - if encoded_bytes(&request) == target { - return request; +fn message_for(request: &HuginnMindRequest, source: Option) -> CultNetMessage { + encode_request("eureka-state-call", request, source).unwrap() +} + +/// The smallest session padding whose request encodes to at least `target` +/// bytes. The payload rides as base64, so request sizes step by four and only +/// every fourth size is reachable through the request alone. +fn pad_for(target: usize) -> usize { + let (mut low, mut high) = (0, target); + while low < high { + let middle = (low + high) / 2; + if encoded_bytes(&message_for(&padded_admit(middle), None)) < target { + low = middle + 1; + } else { + high = middle; } } - panic!("no padding encodes to {target} bytes"); + low +} + +/// The largest request one send carries, and the smallest it does not. +fn largest_request_that_fits() -> HuginnMindRequest { + padded_admit(pad_for(MAX_REQUEST_BYTES + 1) - 1) +} + +fn smallest_request_over() -> HuginnMindRequest { + padded_admit(pad_for(MAX_REQUEST_BYTES + 1)) +} + +/// A message that encodes to exactly `target` bytes: the largest request that +/// fits under it, topped up with a source runtime id, which rides outside the +/// payload and moves the size a byte at a time. +fn message_of_size(target: usize) -> CultNetMessage { + let request = padded_admit(pad_for(target + 1) - 1); + for pad in 0..16 { + let message = message_for(&request, (pad > 0).then(|| "r".repeat(pad))); + if encoded_bytes(&message) == target { + return message; + } + } + panic!("no message encodes to {target} bytes"); } fn admitted_answer() -> CultNetMessage { @@ -429,26 +449,27 @@ fn admitted_answer() -> CultNetMessage { /// A request larger than one send carries is refused by the client before it /// connects to anything, as its own error and not as a transport failure the -/// caller would retry; one byte under the limit is sent, and answered. +/// caller would retry; the largest request that fits is sent, and answered. #[test] -fn a_request_over_what_one_send_carries_is_too_large_and_one_at_the_limit_is_sent() { +fn a_request_over_what_one_send_carries_is_too_large_and_one_that_fits_is_sent() { let closed = UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap(); - let over = request_of_size(MAX_REQUEST_BYTES + 1); + let over = smallest_request_over(); let started = Instant::now(); - match client(closed).call(over) { - Err(ClientError::TooLarge { bytes, limit }) => assert_eq!((bytes, limit), (MAX_REQUEST_BYTES + 1, MAX_REQUEST_BYTES)), + match client(closed).call(over.clone()) { + Err(ClientError::TooLarge { bytes, limit }) => { + assert_eq!(bytes, encoded_bytes(&message_for(&over, None))); + assert_eq!(limit, MAX_REQUEST_BYTES); + assert!(bytes > limit); + } other => panic!("expected TooLarge, got {other:?}"), } assert!(started.elapsed() < Duration::from_secs(1), "refused only after trying the transport"); + let fits = largest_request_that_fits(); + assert!(MAX_REQUEST_BYTES - encoded_bytes(&message_for(&fits, None)) < 4, "not the largest that fits"); let server = answering(admitted_answer()); - for size in [MAX_REQUEST_BYTES - 1, MAX_REQUEST_BYTES] { - let answer = client(server.addr).call(request_of_size(size)); - assert!( - matches!(answer, Ok(HuginnMindResponse::Admit(PipelineAdmissionOutcome::AlreadyAdmitted { .. }))), - "a request of {size} bytes: {answer:?}" - ); - } + let answer = client(server.addr).call(fits); + assert!(matches!(answer, Ok(HuginnMindResponse::Admit(PipelineAdmissionOutcome::AlreadyAdmitted { .. }))), "{answer:?}"); } /// The transport's own refusal of an oversize send ("Message too long") is @@ -457,6 +478,30 @@ fn a_request_over_what_one_send_carries_is_too_large_and_one_at_the_limit_is_sen #[test] fn an_oversize_admission_is_never_reported_as_an_unavailable_daemon() { let server = answering(admitted_answer()); - let error = client(server.addr).call(padded_admit(6, 70_000)).unwrap_err(); + let error = client(server.addr).call(padded_admit(70_000)).unwrap_err(); assert!(matches!(error, ClientError::TooLarge { .. }), "{error}"); } + +/// The limit is the transport's own: a message of `MAX_REQUEST_BYTES` goes out +/// on an RUDP session, and one byte more is refused by the socket. +#[test] +fn the_limit_is_the_largest_message_the_transport_sends() { + let server = answering(admitted_answer()); + let mut session = CultMesh::create_rudp_client_for_endpoint( + "limit-probe".to_string(), + CULTNET_OPERATION_CONNECTION_ID, + &format!("rudp://{}", server.addr), + CultMeshRudpSocketOptions::default(), + ) + .unwrap(); + session.connect(Vec::new()).unwrap(); + let until = Instant::now() + Duration::from_secs(5); + while !session.connected() { + session.poll_resends().unwrap(); + let _ = session.receive_once().unwrap(); + assert!(Instant::now() < until, "the server never accepted"); + std::thread::sleep(Duration::from_millis(2)); + } + session.send_schema_message(&message_of_size(MAX_REQUEST_BYTES)).expect("the limit is carried"); + assert!(session.send_schema_message(&message_of_size(MAX_REQUEST_BYTES + 1)).is_err(), "one byte more is sent"); +} From c2b2952749e4cfafcefc304dee7fa72262358b89 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 03:04:04 +0200 Subject: [PATCH 039/520] eureka-state tests: reach the limit through the source runtime id, seed the stewardship The source runtime id adds its field name when present, so exact sizes are searched over both paddings. The attribution test admits a campaign, which needs its repo stewarded first. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/tests/client.rs | 19 +++++++++++-------- crates/eureka-state/tests/mcp.rs | 4 ++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs index ff7d50c..6a054ea 100644 --- a/crates/eureka-state/tests/client.rs +++ b/crates/eureka-state/tests/client.rs @@ -423,15 +423,18 @@ fn smallest_request_over() -> HuginnMindRequest { padded_admit(pad_for(MAX_REQUEST_BYTES + 1)) } -/// A message that encodes to exactly `target` bytes: the largest request that -/// fits under it, topped up with a source runtime id, which rides outside the -/// payload and moves the size a byte at a time. +/// A message that encodes to exactly `target` bytes: a request a little under +/// it, topped up with a source runtime id, which rides outside the payload and +/// moves the size a byte at a time once the field is there at all. fn message_of_size(target: usize) -> CultNetMessage { - let request = padded_admit(pad_for(target + 1) - 1); - for pad in 0..16 { - let message = message_for(&request, (pad > 0).then(|| "r".repeat(pad))); - if encoded_bytes(&message) == target { - return message; + let fitting = pad_for(target + 1) - 1; + for padding in (fitting.saturating_sub(80)..=fitting).rev() { + let request = padded_admit(padding); + for source in 1..80 { + let message = message_for(&request, Some("r".repeat(source))); + if encoded_bytes(&message) == target { + return message; + } } } panic!("no message encodes to {target} bytes"); diff --git a/crates/eureka-state/tests/mcp.rs b/crates/eureka-state/tests/mcp.rs index f6def4c..3a51eb0 100644 --- a/crates/eureka-state/tests/mcp.rs +++ b/crates/eureka-state/tests/mcp.rs @@ -413,7 +413,7 @@ fn an_oversize_admit_is_a_typed_too_large_error() { /// is a typed refusal, and nothing is committed. #[test] fn admit_refuses_an_unusable_attribution() { - let (root, daemon) = mind(vec![]); + let (root, daemon) = mind(vec![vec![stewardship()]]); let server = serve(root, daemon); let mut mcp = Mcp::at(INSTANCE, server.addr); for (field, value) in [ @@ -437,7 +437,7 @@ fn admit_refuses_an_unusable_attribution() { ); } let (_, whoami) = mcp.call("whoami", json!({})); - assert_eq!(whoami["status"]["receipts"], json!(1), "only the seed is committed: {whoami}"); + assert_eq!(whoami["status"]["receipts"], json!(2), "only the seed is committed: {whoami}"); let (error, admitted) = mcp.call("admit", faculty_args(vec![campaign_json()])); assert!(!error); assert!(admitted.get("Committed").is_some(), "{admitted}"); From 3d9965983f4dc1695eebd2f8e23b725a3f767bf0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 04:55:04 +0200 Subject: [PATCH 040/520] huginn-mind: provenance follows the leaf's Title rule and covers the tool refuse_provenance validated only agent and session, and only for blank and a handful of line breaks. It now covers agent, session and tool with the leaf's Title character rule (an alphanumeric character, no control or line-break character, no bidi control) inside Short's byte bound. The leaf keeps that rule private, so a differential test holds the copy to a campaign Title. Restores the stage doc comment that drifted above SHORT_MAX. Multibyte bound test kills a chars-count mutant. Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/admission.rs | 136 ++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 38 deletions(-) diff --git a/crates/huginn-mind/src/admission.rs b/crates/huginn-mind/src/admission.rs index 0288ad0..6443e94 100644 --- a/crates/huginn-mind/src/admission.rs +++ b/crates/huginn-mind/src/admission.rs @@ -172,31 +172,40 @@ impl Mind { } } -/// A3 through the leaf, then A5. /// The bound `epiphany_pipeline::Short` declares, in UTF-8 bytes. The leaf /// exposes no validator for it, so `short_bound_is_the_leafs_own` pins this to /// the `maxLength` its schema publishes. const SHORT_MAX: usize = 200; /// Who admitted a batch is the receipt's only record of it, so the mind checks -/// it here and every writer, whatever its transport, gets the check: the agent -/// and the session are within `Short`'s bound, hold something besides -/// whitespace, and sit on one line. Refused as the leaf's own refusals, since -/// the fields are the leaf's own type. +/// it here and every writer, whatever its transport, gets the check: the agent, +/// the session and the tool follow the leaf's `Title` rule (one alphanumeric +/// character, no control or line-break character, no bidi control) within +/// `Short`'s bound. The leaf keeps that rule private, so +/// `provenance_text_is_the_leafs_title_rule` pins this to it. Refused as the +/// leaf's own refusals, since the fields are the leaf's own type. fn refuse_provenance(provenance: &PipelineProvenance) -> Result<(), MindRefusal> { - for (field, value) in [("provenance.agent", &provenance.agent), ("provenance.session", &provenance.session)] { + let bidi = |c: char| { + matches!(c, '\u{061C}' | '\u{200E}' | '\u{200F}' | '\u{202A}'..='\u{202E}' | '\u{2066}'..='\u{2069}') + }; + let control = |c: char| c.is_control() || matches!(c, '\u{2028}' | '\u{2029}'); + for (field, value) in [ + ("provenance.agent", &provenance.agent), + ("provenance.session", &provenance.session), + ("provenance.tool", &provenance.tool), + ] { let text = value.0.as_str(); + if !text.chars().any(char::is_alphanumeric) || text.contains(control) || text.contains(bidi) { + return Err(PipelineRefusal::InvalidFormat { field: field.into(), value: text.into() }.into()); + } if text.len() > SHORT_MAX { return Err(PipelineRefusal::FieldBound { field: field.into(), limit: SHORT_MAX as u32, actual: text.len() as u32 }.into()); } - let line_break = |c: char| matches!(c, '\n' | '\r' | '\u{0B}' | '\u{0C}' | '\u{85}' | '\u{2028}' | '\u{2029}'); - if text.trim().is_empty() || text.contains(line_break) { - return Err(PipelineRefusal::InvalidFormat { field: field.into(), value: text.into() }.into()); - } } Ok(()) } +/// A3 through the leaf, then A5. fn stage(envelope: CultCacheEnvelope, mind: &Slug) -> Result { validate_pipeline_write_envelope(&envelope)?; let document = PipelineDocument::decode(&envelope)?; @@ -720,8 +729,12 @@ mod tests { } fn admit_attributed(agent: &str, session: &str) -> (Mind, PipelineAdmissionOutcome) { + admit_signed(agent, session, "admit") + } + + fn admit_signed(agent: &str, session: &str, tool: &str) -> (Mind, PipelineAdmissionOutcome) { let mut mind = opened(MemoryStore::new(), INSTANCE); - let provenance = PipelineProvenance { faculty: Faculty::Hands, agent: s(agent), session: s(session), tool: s("admit") }; + let provenance = PipelineProvenance { faculty: Faculty::Hands, agent: s(agent), session: s(session), tool: s(tool) }; let batch = PipelineAdmissionBatch { instance: slug(INSTANCE), provenance, documents: vec![instance(INSTANCE), stewardship(INSTANCE, REPO)] }; let outcome = mind.admit(batch, now()); (mind, outcome) @@ -731,45 +744,92 @@ mod tests { /// a blank, multi-line or oversized attribution, and a refused batch leaves /// nothing behind. #[test] - fn provenance_names_an_agent_and_a_session_on_one_line_within_short() { + fn provenance_names_an_agent_a_session_and_a_tool_on_one_clean_line_within_short() { let over = "x".repeat(SHORT_MAX + 1); let exact = "y".repeat(SHORT_MAX); - for (field, agent, session) in [ - ("provenance.agent", "", "session-1"), - ("provenance.agent", " \t ", "session-1"), - ("provenance.session", "claude", ""), - ("provenance.session", "claude", " "), - ("provenance.session", "claude", "\u{2003}"), - ("provenance.session", "claude", "line\nbreak"), - ("provenance.session", "claude", "line\rbreak"), - ("provenance.session", "claude", "a\u{2028}b"), - ("provenance.agent", "two\nlines", "session-1"), + for bad in [ + "", + " \t ", + "\u{2003}", + "\u{200B}", + "line\nbreak", + "line\rbreak", + "a\u{0B}b", + "a\u{0C}b", + "a\u{85}b", + "a\u{2028}b", + "a\u{2029}b", + "a\0b", + "a\u{1B}[31mb", + "a\u{202E}b", + "a\u{2066}b", + "a\u{200F}b", ] { - let (mind, outcome) = admit_attributed(agent, session); - let refused = refusal(outcome); - assert!( - matches!(&refused, MindRefusal::Document(PipelineRefusal::InvalidFormat { field: named, .. }) if named == field), - "{agent:?}/{session:?}: {refused:?}" - ); - assert!(mind.is_empty(), "{agent:?}/{session:?}: a refused batch wrote"); + for field in ["provenance.agent", "provenance.session", "provenance.tool"] { + let (mind, outcome) = match field { + "provenance.agent" => admit_signed(bad, "session-1", "admit"), + "provenance.session" => admit_signed("claude", bad, "admit"), + _ => admit_signed("claude", "session-1", bad), + }; + let refused = refusal(outcome); + assert!( + matches!(&refused, MindRefusal::Document(PipelineRefusal::InvalidFormat { field: named, .. }) if named == field), + "{field}={bad:?}: {refused:?}" + ); + assert!(mind.is_empty(), "{field}={bad:?}: a refused batch wrote"); + } } - let (mind, outcome) = admit_attributed(&over, "session-1"); + for (field, outcome) in [ + ("provenance.agent", admit_signed(&over, "session-1", "admit")), + ("provenance.session", admit_signed("claude", &over, "admit")), + ("provenance.tool", admit_signed("claude", "session-1", &over)), + ] { + let (mind, outcome) = outcome; + assert_eq!( + refusal(outcome), + MindRefusal::Document(PipelineRefusal::FieldBound { field: field.into(), limit: 200, actual: 201 }) + ); + assert!(mind.is_empty()); + } + // The bound is in bytes, not characters: 101 two-byte characters are 202 bytes. + let (_, outcome) = admit_attributed(&"é".repeat(101), "session-1"); assert_eq!( refusal(outcome), - MindRefusal::Document(PipelineRefusal::FieldBound { field: "provenance.agent".into(), limit: 200, actual: 201 }) - ); - assert!(mind.is_empty()); - let (_, outcome) = admit_attributed("claude", &over); - assert_eq!( - refusal(outcome), - MindRefusal::Document(PipelineRefusal::FieldBound { field: "provenance.session".into(), limit: 200, actual: 201 }) + MindRefusal::Document(PipelineRefusal::FieldBound { field: "provenance.agent".into(), limit: 200, actual: 202 }) ); + let (_, outcome) = admit_signed("claude", "session-1", &"é".repeat(101)); + assert!(matches!(refusal(outcome), MindRefusal::Document(PipelineRefusal::FieldBound { actual: 202, .. }))); + let (_, outcome) = admit_attributed(&"é".repeat(100), "session-1"); + committed(outcome); // The bound is inclusive and a name with inner spaces is a name. - let (mind, outcome) = admit_attributed(&exact, "session 7"); + let (mind, outcome) = admit_signed(&exact, "session 7", "admit tool"); committed(outcome); let receipts = mind.receipts().unwrap(); assert_eq!(receipts[0].provenance.agent, s(&exact)); assert_eq!(receipts[0].provenance.session, s("session 7")); + assert_eq!(receipts[0].provenance.tool, s("admit tool")); + } + + /// The leaf's `Title` rule is private, so this holds the provenance rule to + /// it: the two accept and refuse the same text. + #[test] + fn provenance_text_is_the_leafs_title_rule() { + let leaf_accepts = |text: &str| { + let D::Campaign(mut campaign) = campaign(&[REPO]) else { unreachable!() }; + campaign.title = epiphany_pipeline::Title(text.into()); + D::Campaign(campaign).validate().is_ok() + }; + let mine_accepts = |text: &str| { + let provenance = PipelineProvenance { faculty: Faculty::Hands, agent: s(text), session: s("s"), tool: s("t") }; + refuse_provenance(&provenance).is_ok() + }; + for text in [ + "claude", "session 7", "", " ", "\t", "\u{200B}", "\u{2003}", "!!!", "a\nb", "a\u{0B}b", "a\u{85}b", "a\u{2028}b", + "a\0b", "a\u{7F}b", "a\u{202E}b", "a\u{061C}b", "a\u{2069}b", "a\u{200D}b", "é", "日本", "a\u{FEFF}b", + &"x".repeat(200), &"x".repeat(201), &"é".repeat(100), &"é".repeat(101), + ] { + assert_eq!(mine_accepts(text), leaf_accepts(text), "{text:?}"); + } } #[test] From cd3dfc0268210736f059cb7501173ec2a36ad7ef Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 04:55:05 +0200 Subject: [PATCH 041/520] eureka-state: an encode failure is not retryable, and a non-UTF-8 endpoint is pinned A request that fails to encode is a defect in this client, not a transport failure, so it is ClientError::Unencodable, typed Unencodable at the tool, instead of the retryable Unavailable. A test reports a non-UTF-8 HUGINN_ENDPOINT as such. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/src/lib.rs | 12 ++++++++---- crates/eureka-state/src/main.rs | 5 +++++ crates/eureka-state/tests/client.rs | 4 ++-- crates/eureka-state/tests/mcp.rs | 13 +++++++++++++ 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/crates/eureka-state/src/lib.rs b/crates/eureka-state/src/lib.rs index 1ff2bb7..b6208e8 100644 --- a/crates/eureka-state/src/lib.rs +++ b/crates/eureka-state/src/lib.rs @@ -45,13 +45,15 @@ pub const MAX_REQUEST_BYTES: usize = 65_507 - 36 - "schema".len(); /// the only one worth retrying: the daemon could not be reached, went quiet /// past the call's deadline, or answered out of protocol. `Rejected` is the /// daemon refusing the envelope itself, and `TooLarge` is the client refusing -/// a request that no single send can carry, before sending it; a retry of the -/// same call cannot change either. +/// a request that no single send can carry, before sending it; `Unencodable` is +/// the request failing to encode at all, a defect in this client. A retry of the +/// same call cannot change any of those three. #[derive(Debug)] pub enum ClientError { Unavailable { endpoint: SocketAddr, detail: String }, Rejected { endpoint: SocketAddr, code: String, detail: String }, TooLarge { bytes: usize, limit: usize }, + Unencodable { detail: String }, } impl fmt::Display for ClientError { @@ -66,6 +68,7 @@ impl fmt::Display for ClientError { Self::TooLarge { bytes, limit } => { write!(f, "the request encodes to {bytes} bytes and one send carries at most {limit}") } + Self::Unencodable { detail } => write!(f, "the request could not be encoded: {detail}"), } } } @@ -109,9 +112,10 @@ impl HuginnClient { let deadline = Instant::now() + self.timeout; let unavailable = |error: anyhow::Error| ClientError::Unavailable { endpoint: self.endpoint, detail: format!("{error:#}") }; - let message = encode_request(MESSAGE_ID, &request, None).map_err(unavailable)?; + let unencodable = |error: anyhow::Error| ClientError::Unencodable { detail: format!("{error:#}") }; + let message = encode_request(MESSAGE_ID, &request, None).map_err(unencodable)?; let bytes = encode_cultnet_message_to_vec(&message, CultNetWireContract::CultNetSchemaV0) - .map_err(unavailable)? + .map_err(unencodable)? .len(); if bytes > MAX_REQUEST_BYTES { return Err(ClientError::TooLarge { bytes, limit: MAX_REQUEST_BYTES }); diff --git a/crates/eureka-state/src/main.rs b/crates/eureka-state/src/main.rs index d5a8bc5..d368a23 100644 --- a/crates/eureka-state/src/main.rs +++ b/crates/eureka-state/src/main.rs @@ -108,6 +108,7 @@ impl Trouble { Self::Client(ClientError::TooLarge { bytes, limit }) => { json!({ "error": "TooLarge", "bytes": bytes, "limit": limit }) } + Self::Client(ClientError::Unencodable { detail }) => json!({ "error": "Unencodable", "detail": detail }), Self::Internal(detail) => json!({ "error": "Internal", "detail": detail }), } } @@ -317,6 +318,10 @@ mod tests { Trouble::Client(ClientError::TooLarge { bytes: 70_000, limit: 65_000 }).body(), json!({ "error": "TooLarge", "bytes": 70_000, "limit": 65_000 }) ); + assert_eq!( + Trouble::Client(ClientError::Unencodable { detail: "no".into() }).body(), + json!({ "error": "Unencodable", "detail": "no" }) + ); assert_eq!(Trouble::Misconfigured("bad".into()).body(), json!({ "error": "Misconfigured", "detail": "bad" })); } diff --git a/crates/eureka-state/tests/client.rs b/crates/eureka-state/tests/client.rs index 6a054ea..3f34512 100644 --- a/crates/eureka-state/tests/client.rs +++ b/crates/eureka-state/tests/client.rs @@ -128,14 +128,14 @@ fn client(addr: SocketAddr) -> HuginnClient { fn unavailable(error: ClientError) -> (SocketAddr, String) { match error { ClientError::Unavailable { endpoint, detail } => (endpoint, detail), - other @ (ClientError::Rejected { .. } | ClientError::TooLarge { .. }) => panic!("expected Unavailable, got {other}"), + other @ (ClientError::Rejected { .. } | ClientError::TooLarge { .. } | ClientError::Unencodable { .. }) => panic!("expected Unavailable, got {other}"), } } fn rejected(error: ClientError) -> (SocketAddr, String, String) { match error { ClientError::Rejected { endpoint, code, detail } => (endpoint, code, detail), - other @ (ClientError::Unavailable { .. } | ClientError::TooLarge { .. }) => panic!("expected Rejected, got {other}"), + other @ (ClientError::Unavailable { .. } | ClientError::TooLarge { .. } | ClientError::Unencodable { .. }) => panic!("expected Rejected, got {other}"), } } diff --git a/crates/eureka-state/tests/mcp.rs b/crates/eureka-state/tests/mcp.rs index 3a51eb0..27ee9fd 100644 --- a/crates/eureka-state/tests/mcp.rs +++ b/crates/eureka-state/tests/mcp.rs @@ -495,6 +495,19 @@ fn a_non_utf8_instance_is_reported_as_such() { assert_eq!(whoami["instance"], Value::Null); } +/// Likewise an endpoint that is not UTF-8: reported as that, not as unset. +#[test] +fn a_non_utf8_endpoint_is_reported_as_such() { + let mut mcp = Mcp::spawn(|command| { + command.env("EUREKA_INSTANCE", INSTANCE).env("HUGINN_ENDPOINT", non_utf8()); + }); + let (error, whoami) = mcp.call("whoami", json!({})); + assert!(!error); + let detail = whoami["error"]["detail"].as_str().unwrap(); + assert!(detail.contains("HUGINN_ENDPOINT is not valid UTF-8"), "{whoami}"); + assert_eq!(whoami["endpoint"], Value::Null); +} + #[cfg(unix)] fn non_utf8() -> std::ffi::OsString { std::os::unix::ffi::OsStringExt::from_vec(vec![b'e', 0xff]) From dcb85def310e21057d9e2e7e9aede1d57a4896a0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 13:25:55 +0200 Subject: [PATCH 042/520] README: the daemon's full run command and where it is deployed --qdrant-url, --ollama-url and --embedding-model are required since Cut 11a; the example now runs. Point at the gamecult-ops runbook for the Yggdrasil unit. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 985f9d3..b6a4bd6 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,13 @@ next. ```powershell cargo check --workspace -cargo run -p huginn-daemon -- --state-root --instance --bind 127.0.0.1:17872 +cargo run -p huginn-daemon -- --state-root --instance --bind 127.0.0.1:17872 ` + --qdrant-url http://127.0.0.1:6333 --ollama-url http://10.77.0.4:11434 --embedding-model qwen3-embedding:0.6b ``` +The deployed daemon (instance `eureka` on Yggdrasil) is installed and run by +`gamecult-ops/runbooks/huginn-yggdrasil.md`. + ## Persona Huginn's legacy repo Persona lives under `.voidbot/`. Its `state/huginn.cc` From 9dd6d8007ab7de84c926a34edd3947ba15d1f01c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Wed, 30 Sep 2026 14:36:44 +0200 Subject: [PATCH 043/520] Pin CultLib 8fc74c7 in every crate, and the pipeline leaf that follows it eureka-state on Windows could not reach the daemon: the cultnet-rs RUDP client bound 127.0.0.1, and a loopback-bound socket cannot send to 10.77.0.1 (os error 10051). CultLib 8fc74c7 binds the unspecified address for a remote endpoint. The daemon and the client move together, because the ack cuts between the revs changed the RUDP handshake. epiphany-pipeline moves to Epiphany 73edee5, which pins the same CultLib, so one cultcache-rs is in the graph and the pipeline's CultCache types stay Huginn's. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 137 ++++++++++++++++++++++++++++++-- crates/eureka-state/Cargo.toml | 2 +- crates/huginn-daemon/Cargo.toml | 2 +- crates/huginn-mind/Cargo.toml | 6 +- 4 files changed, 136 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71de095..7254c78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "version_check", +] + [[package]] name = "android_system_properties" version = "0.1.6" @@ -81,6 +92,12 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.2" @@ -192,7 +209,7 @@ dependencies = [ [[package]] name = "cultcache-rs" version = "0.2.0" -source = "git+https://github.com/GameCult/CultLib.git?rev=6251cbcd37c12ef3f0c1cc50063a03369225b7ac#6251cbcd37c12ef3f0c1cc50063a03369225b7ac" +source = "git+https://github.com/GameCult/CultLib.git?rev=8fc74c709444b884a2bd339e0c306bf9484d99d2#8fc74c709444b884a2bd339e0c306bf9484d99d2" dependencies = [ "anyhow", "chrono", @@ -210,7 +227,7 @@ dependencies = [ [[package]] name = "cultcache-rs-derive" version = "0.1.0" -source = "git+https://github.com/GameCult/CultLib.git?rev=6251cbcd37c12ef3f0c1cc50063a03369225b7ac#6251cbcd37c12ef3f0c1cc50063a03369225b7ac" +source = "git+https://github.com/GameCult/CultLib.git?rev=8fc74c709444b884a2bd339e0c306bf9484d99d2#8fc74c709444b884a2bd339e0c306bf9484d99d2" dependencies = [ "proc-macro2", "quote", @@ -220,7 +237,7 @@ dependencies = [ [[package]] name = "cultnet-rs" version = "0.1.0" -source = "git+https://github.com/GameCult/CultLib.git?rev=6251cbcd37c12ef3f0c1cc50063a03369225b7ac#6251cbcd37c12ef3f0c1cc50063a03369225b7ac" +source = "git+https://github.com/GameCult/CultLib.git?rev=8fc74c709444b884a2bd339e0c306bf9484d99d2#8fc74c709444b884a2bd339e0c306bf9484d99d2" dependencies = [ "aes-gcm", "anyhow", @@ -230,6 +247,7 @@ dependencies = [ "ed25519-dalek", "hmac", "rand", + "reed-solomon-erasure", "rmp-serde", "rmpv", "serde", @@ -357,7 +375,7 @@ dependencies = [ [[package]] name = "epiphany-pipeline" version = "0.1.0" -source = "git+https://github.com/GameCult/Epiphany.git?rev=4891f2e59008efeb262197564eddaa75bc3c75ea#4891f2e59008efeb262197564eddaa75bc3c75ea" +source = "git+https://github.com/GameCult/Epiphany.git?rev=73edee5cdaddece8bda9c6c3271e8ddd064337f9#73edee5cdaddece8bda9c6c3271e8ddd064337f9" dependencies = [ "anyhow", "chrono", @@ -565,6 +583,15 @@ dependencies = [ "polyval", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + [[package]] name = "hmac" version = "0.12.1" @@ -665,6 +692,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "itoa" version = "1.0.18" @@ -688,18 +724,42 @@ version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "linux-raw-sys" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown", +] + [[package]] name = "memchr" version = "2.8.3" @@ -727,6 +787,31 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + [[package]] name = "pastey" version = "0.2.3" @@ -853,6 +938,28 @@ dependencies = [ "libc", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "reed-solomon-erasure" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f" +dependencies = [ + "libm", + "lru", + "parking_lot", + "smallvec", + "spin", +] + [[package]] name = "ref-cast" version = "1.0.27" @@ -952,7 +1059,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", @@ -991,6 +1098,12 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "semver" version = "1.0.28" @@ -1113,6 +1226,12 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + [[package]] name = "socket2" version = "0.5.10" @@ -1123,6 +1242,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + [[package]] name = "spki" version = "0.7.3" @@ -1174,7 +1299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/crates/eureka-state/Cargo.toml b/crates/eureka-state/Cargo.toml index be76e34..9a0d37a 100644 --- a/crates/eureka-state/Cargo.toml +++ b/crates/eureka-state/Cargo.toml @@ -15,7 +15,7 @@ publish.workspace = true # consumer: four tools over stdio, one per wire operation, holding no state. [dependencies] anyhow = "1" -cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "6251cbcd37c12ef3f0c1cc50063a03369225b7ac", package = "cultnet-rs" } +cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultnet-rs" } huginn-mind = { path = "../huginn-mind" } rmcp = { version = "2.2.0", default-features = false, features = ["server", "macros", "transport-io"] } rmp-serde = "1" diff --git a/crates/huginn-daemon/Cargo.toml b/crates/huginn-daemon/Cargo.toml index 5890c67..b2cb3fd 100644 --- a/crates/huginn-daemon/Cargo.toml +++ b/crates/huginn-daemon/Cargo.toml @@ -21,7 +21,7 @@ path = "src/main.rs" anyhow = "1" base64 = "0.22" chrono = "0.4.44" -cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "6251cbcd37c12ef3f0c1cc50063a03369225b7ac", package = "cultnet-rs" } +cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultnet-rs" } huginn-mind = { path = "../huginn-mind" } rmp-serde = "1" serde = { version = "1", features = ["derive"] } diff --git a/crates/huginn-mind/Cargo.toml b/crates/huginn-mind/Cargo.toml index ccf4e94..883e9b0 100644 --- a/crates/huginn-mind/Cargo.toml +++ b/crates/huginn-mind/Cargo.toml @@ -17,9 +17,9 @@ publish.workspace = true anyhow = "1" base64 = "0.22" chrono = "0.4.44" -cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "6251cbcd37c12ef3f0c1cc50063a03369225b7ac", package = "cultcache-rs" } -cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "6251cbcd37c12ef3f0c1cc50063a03369225b7ac", package = "cultnet-rs" } -epiphany-pipeline = { git = "https://github.com/GameCult/Epiphany.git", rev = "4891f2e59008efeb262197564eddaa75bc3c75ea", package = "epiphany-pipeline" } +cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultcache-rs" } +cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultnet-rs" } +epiphany-pipeline = { git = "https://github.com/GameCult/Epiphany.git", rev = "73edee5cdaddece8bda9c6c3271e8ddd064337f9", package = "epiphany-pipeline" } rmp-serde = "1" schemars = "1" serde = { version = "1", features = ["derive"] } From 133324a7f78047164f0ca02331398066a971ac15 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 10:31:23 +0200 Subject: [PATCH 044/520] Eureka substrate target: every subject resolvable, refusals typed, one gap home Campaign eureka-substrate, the Mind fixes the operator ruled next after idunn-watchdog. Typed target and rulings live in Eureka's mind. Co-Authored-By: Claude Opus 5.5 --- docs/eureka-substrate-target.md | 84 +++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 docs/eureka-substrate-target.md diff --git a/docs/eureka-substrate-target.md b/docs/eureka-substrate-target.md new file mode 100644 index 0000000..30d53ce --- /dev/null +++ b/docs/eureka-substrate-target.md @@ -0,0 +1,84 @@ +# Eureka substrate: target + +Status: target, Self, 2026-10-01. Campaign `eureka-substrate` in Eureka's mind +(instance `eureka`). Its questions, rulings, cut specs, verification and the +substrate gaps themselves are typed documents there. This page keeps the +rationale and the design truths the campaign produces. The cut map's body facts +and model page will live in `eureka-substrate-map.md`. + +## Why + +The first typed campaign, `idunn-watchdog` (postmortem: Idunn +`docs/watchdog-reporting-postmortem.md`, `6667f0f`), ran its whole pipeline +through Huginn and found the substrate's defects by hitting them. The operator +ruled, 2026-10-01: + +> Yep, fixing Mind issues is highest priority, immediately once this campaign +> closes + +The defects that cost the most: + +- **A finding can be admitted that can never be resolved.** A key's local part + is bounded at 64 bytes as a whole, and only when it is composed + (`epiphany-pipeline` `LOCAL_MAX`). A resolution's local embeds its subject's + local plus `finding.`/`follow_up.` and `.n`. So a subject whose own key + fits can have no admissible resolution, under any outcome. Six findings in + `idunn-watchdog` are stuck in force this way. +- **A shape refusal arrives labelled `Unavailable`.** Huginn's admission maps + the leaf's key error through `anyhow` into `MindRefusal::Unavailable`. The + skill tells every agent that `Unavailable` means the organ is down: retry + once, then stop. A faithful agent would halt a campaign over a key that was + too long. +- **Substrate gaps have no home every campaign can see.** Follow-ups are + campaign-rooted, so 20 `gap-*` follow-ups sit under `idunn-watchdog`, and a + later campaign's recipe cannot see them. +- **Smaller defects:** + - Revising one spec forced a revision of every spec that named it in + `depends_on`, twice, when the dependency was on a cut. + - A short sha and the full sha of one commit are different values to + admission. + - A finding cannot be deferred to the cut that fixes it, only to a pointer + follow-up. + - The faculty is named Life, but the enum says `MindSteward`. + - Stewardship is universal (operator, 2026-09-30), yet admission demands a + per-repo stewardship record that decides nothing. + +The code the live mind runs sits on two long-lived branches: Huginn +`eureka/memory-organ` (release `9dd6d80`), and Epiphany +`codex/eureka-pipeline-state`, where the leaf Huginn pins lives. The operator +ruled that they merge to main first. + +## Design truths this campaign produces + +- **Every admitted subject stays resolvable.** If admission accepts a document + whose kind can be resolved, every outcome its subject allows can be admitted + for it, up to a stated sequence bound. A grammar that cannot hold a key + refuses the subject at admission, not its resolution later. +- **A refusal names what was wrong.** A document that breaks a shape, bound or + key rule is refused as a typed document refusal. `Unavailable` means only + that the organ could not serve. +- **Stored documents stay valid.** No change in this campaign re-keys, + re-encodes or invalidates a document already in the mind. The epoch stays + `epiphany.pipeline.epoch.v2`. Every change is a widening, an added variant, + or an admission rule that judges new batches only. +- **One home for substrate gaps.** This campaign is the standing home for gaps + in the mind's own substrate. The skill's recipe names its root, and every + campaign reads it. +- **The mind's code comes from main.** The Huginn daemon, the leaf and the + `eureka-state` client the live mind runs are built from commits on Huginn's + and Epiphany's main. +- **The live mind survives its own upgrade.** Each deploy upgrades + `/var/lib/gamecult/huginn/minds/eureka/mind.redb` in place under a release + symlink that can be flipped back. Every document admitted before the upgrade + reads back after it. + +## Not this campaign + +- **Verify-stopgap gaps** (artifact return, `rg` in the image, sibling repos): + Idunn's verify campaign. +- **CultNet selection** (`gap-citer-faculty-hop`): CultLib. +- **The cold-embed timeout** and **a session-id tool**: recorded follow-ups. +- **Leasing stewardship between agents and `hand_off`**: Huginn Cut 12, parked. +- **Re-encoding stewardship ids into a readable form**, which would re-key + stored documents. +- **Persona-state reading in Huginn.** From f97bcedd7bc14c8683ca04e8d288b6772bc93d1a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 10:57:30 +0200 Subject: [PATCH 045/520] Eureka substrate map: body facts B1-B6, model page, rationale The id parser bounds locals at 64 too; Faculty lives in Huginn; two refusal sites map key errors to Unavailable; six findings unresolvable and 39 resolutions unwithdrawable in the live mind; the merges are safe. Co-Authored-By: Claude Opus 5.5 --- docs/eureka-substrate-map.md | 255 +++++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 docs/eureka-substrate-map.md diff --git a/docs/eureka-substrate-map.md b/docs/eureka-substrate-map.md new file mode 100644 index 0000000..abeff46 --- /dev/null +++ b/docs/eureka-substrate-map.md @@ -0,0 +1,255 @@ +# Eureka substrate: map + +Status: map, Imagination (`imagination-substrate`), 2026-10-01. Campaign +`eureka-substrate` in Eureka's mind. Questions, cut specs and rulings are +typed documents there; this page keeps body facts, the model page and +rationale only. Target: `docs/eureka-substrate-target.md` (`133324a7`). + +Pinned heads for every fact below: + +| Repo | Ref | Commit | +| --- | --- | --- | +| Huginn | `origin/main` | `133324a7f78047164f0ca02331398066a971ac15` | +| Huginn | `origin/eureka/memory-organ` (deployed release) | `9dd6d8007ab7de84c926a34edd3947ba15d1f01c` | +| Epiphany | `origin/main` | `4d1113ef0a4dbcead2e2c3cdc43aa13ef773dbb8` | +| Epiphany | `origin/codex/eureka-pipeline-state` | `4006b14bf29b49410bd18b43acb4d42c3c4e2443` | +| Epiphany | leaf pinned by Huginn (`crates/huginn-mind/Cargo.toml:22`) | `73edee5cdaddece8bda9c6c3271e8ddd064337f9` | +| CultLib | `cultcache-rs` pinned by both | `8fc74c709444b884a2bd339e0c306bf9484d99d2` | +| gamecult-ops | `main` | `b4eea88d801ad45aad2a49574a9eb2f750c7dd97` | +| Eureka skill | `main` | `094083566e19cccca026de8cbfc24462f1debe93` | + +Source reads were made in scratch clones at the pinned commits +(`scratchpad/imagination-substrate/{epiphany,huginn}`); line numbers are +against those commits. `lib.rs` below is `epiphany-pipeline/src/lib.rs`. + +## Body facts + +### B1. Key grammar (source read, leaf at `73edee5c`) + +- `LOCAL_MAX = 64` (`lib.rs:877`) bounds every composed local, whole + (`local`, `lib.rs:892-901`). It is the only bound on a resolution chain's + depth (comment `lib.rs:917-922`, test `lib.rs:2105-2150`). +- A resolution's local is `..n` + (`lib.rs:923-929`). Overhead over the subject's local: `finding` 11 bytes, + `follow_up` 13, `question` 12, `ruling` 10, `cut_spec` 12, `stewardship` 15 + at one sequence digit. A withdrawal (resolution of a resolution) adds + `resolution.` and `.n`, 14 more. +- So with subjects bounded at 64 and resolutions at 64, a finding local over + 53, a follow-up label over 51, a question label over 52, a ruling label over + 54 or a cut-spec local over 52 admits and can never be resolved. Self's map + said follow-up labels over 52; it is 51 (`follow_up` is 9 bytes, plus two + dots and `n1`). +- `pipeline_id` (`lib.rs:798-819`) parses every id's local with `dotted_text` + (`lib.rs:148-158`), whose 64-byte bound is the same one `Slug` uses + (`lib.rs:251`). Every `PipelineRef` validation goes through it + (`validate`, `lib.rs:589`; `validate_ref`, `lib.rs:609`), and so do + Huginn's view (`query.rs:433`), index (`index.rs:120`) and selection keys and + cites (`rows.rs:306`, `rows.rs:331`). Raising `LOCAL_MAX` alone therefore + makes a key that no id parser accepts. The parser bound has to move with it. +- The epoch is `epiphany.pipeline.epoch.v2` (`lib.rs:886`). `value_types!` + (`lib.rs:400-419`) emits no serde default on any field, so adding a field + would break the decode of stored documents. **No cut in this map adds a + field**, so this constraint is recorded and not exercised. +- The published schemas (`schemas/cultnet/epiphany.pipeline.*.v2.schema.json`) + carry no key bound. A resolution's `subject.id` is `Short`, 200 bytes. A key + is not a field, so no published schema changes when a local bound moves. +- Storage: `cultcache-rs` stores rows in redb as + `TableDefinition<&[u8], &[u8]>` (`CultLib packages/cultcache-rs/src/lib.rs:1023` + at `8fc74c70`), with no key bound of its own. A Qdrant point id is the + SHA-256 of the document id (`huginn-daemon/src/index/mod.rs:225-231`), so + key length does not reach the index. +- Id length ceiling: `::` must fit `Short` (200) + wherever an id is cited. That allows a local of up to 123 bytes. + +### B2. Census of the live mind (probe) + +- Command: `query` for every question, ruling, cut_spec, target, stewardship + and follow_up (119 ids), then every finding and resolution (138 ids), header + projection. The ids were extracted with Python, and the lengths of each + depth-one resolution local (`n1`) and depth-two withdrawal local were + computed. +- Where and when: the eureka-state tools from Starfire, 2026-10-01 about + 08:40 UTC, `as_of` 82, mind status 293 documents, 82 receipts. +- Result (256 resolvable-kind documents): the longest stored local is 64 and + the longest id is 90. + - **Six subjects can have no resolution at all** (n1 local over 64). All + are findings in force under `idunn-watchdog`: + `cut-bifrost-retire-alarm.s2.verb-default-accepts-malformed` (69), + `cut-ops-notice-deploy.s2.pinned-bifrost-predates-retry` (65), + `cut-bifrost-notice-retry.s1.unknown-test-ignores-backoff` (67), + `cut-bifrost-notice-retry.s2.closed-unknown-retry-unpinned` (68), + `cut-bifrost-notice-retry.s2.flapping-clock-retries-every-tick` (72), and + `cut-idunn-topology-lock.s2.boot-reconcile-skipped-on-contention` (74). + That matches Self's count. The seventh finding in force, + `cut-idunn-incident.s3.mutants-share-target-dir`, can be resolved (57), + but its resolution could never be withdrawn (71). + - **39 stored resolutions can never be withdrawn** (their withdrawal local + is 65-78). So their subjects can never be reopened, which is a lifecycle + the matrix allows (`admission.rs:555-560`, Q17 B). Two follow-ups + (`gap-substrate-followups-campaign-rooted` under both roots, and + `projection-mode-branch-vs-private-lock`) and about 40 resolved findings + have the same latent defect one level up. +- Conclusion: an admission guard alone cannot satisfy + `every-subject-resolvable` for stored documents, because they cannot be + re-keyed (`stored-documents-valid`). The resolution's local bound has to + widen. + +### B3. Refusal classification (source read, Huginn at `9dd6d80`) + +- `Mind::admit` (`huginn-mind/src/admission.rs:73-85`) calls + `document.validate()` (which returns a typed refusal), then + `document.prepare(cache)`. `prepare` (`lib.rs:714-721`) runs + `pipeline_key(self)?` into `anyhow`, and admission maps any `prepare` error + through `unavailable` (`mind.rs:46-48`) to `MindRefusal::Unavailable`. The + derived writes take the same path (`admission.rs:136`). A ruling whose + derived `Answered` resolution cannot be keyed is refused `Unavailable` there. +- The daemon passes the outcome through unchanged + (`huginn-daemon/src/daemon.rs:143`). So `Refused(Unavailable{..})` reaches + the client as an admit answer, not as `isError`. +- The later A3 step (`stage` through `validate_pipeline_write_envelope`, + `lib.rs:1003-1015`) returns typed refusals. Only the prepare path is + misclassified. +- `Unavailable` is also raised legitimately, for store and decode faults + (`admission.rs:949`, `admission.rs:2091`, `docs.rs:48`, `mind.rs:140`, + `receipt.rs:209`). Those stay. + +### B4. Smaller rules (source read, Huginn at `9dd6d80`) + +- **Sha identity:** the one place shas are compared is + `admission.rs:433`, `commit.sha == report.range.head`, a literal string + equality. `Sha` is 7-40 lowercase hex (`lib.rs:253`). `FullSha` (40) exists + for `ForeignRef` (`lib.rs:255`, `lib.rs:467`). +- **Deferred:** the matrix (`admission.rs:562-585`) allows + `(Finding, Deferred { to })` only when `to.kind == FollowUp` + (`admission.rs:574`). The leaf's `Deferred { to: PipelineRef }` already takes + any kind, so this is an admission change only. +- **Faculty:** `Faculty` lives in Huginn, not in the leaf + (`huginn-mind/src/receipt.rs:32-40`, variant `MindSteward`). The selection + alias domain is a second copy (`rows.rs:42`). The eureka-state client takes + the type from huginn-mind (`eureka-state/src/main.rs:27`, `main.rs:66`), so + the client must be rebuilt for a new variant. +- Probe: `query` with `faculty any_of [MindSteward]` matched 0 documents + (2026-10-01 08:40 UTC). No stored receipt carries the old name. +- **Stewardship:** admission reads stewardship in one live rule. A campaign + needs a stewardship in force for each repo (`admission.rs:363-366`, + `RepoNotStewarded`). The hand-off rule (`admission.rs:501`, `NotStewarded`) + is parked with Cut 12. Six stewardship documents are stored. +- **depends_on:** this is `Vec[8]` (`lib.rs:525`). Huginn's edge list + says outright that `depends_on` is a label, not an edge + (`huginn-mind/src/docs.rs:398-404`). Nothing checks it. The only reader is + the header projection (`query.rs:121`, `query.rs:173`). The + `idunn-watchdog` specs stored revision ids in it. That came from the skill's + habit, not from the code's intent. +- `open_checked` (`mind.rs:180-188`) re-validates epoch, types and identity, + not keys or bounds, so an opened mind never re-judges a stored key. No + migration path exists, and none is needed under this map. + +### B5. Branches and in-flight work (git, 2026-10-01 ~08:35 UTC) + +- Huginn: `memory-organ` is 107 commits ahead of the merge base `91b7fcf`; + `main` is 1 ahead (`133324a`, docs only). `git merge-tree --write-tree + origin/main origin/eureka/memory-organ` is clean, tree `993ab1fd`. Its only + difference from `9dd6d80` is `docs/eureka-substrate-target.md`. +- Epiphany: the branch is 174 ahead of the merge base `b5d9d4ff`; `main` is 1 + ahead (`4d1113ef`, the Ghostlight pin). Merge-tree is clean, tree + `a75aa388`, and **identical** to the branch tip's tree: the branch already + carries the same Ghostlight pin and lock. The leaf tree `cf84c032` is + identical at `73edee5c`, at the branch tip and in the merge. +- Hands worktrees `hands/rs3` (`9673ab8`, idle since 2026-09-29 06:32), + `hands/cut13a` (`3dc50ee`, 2026-09-30 00:16) and `hands/cut13b` (`cd3dfc0`, + 2026-09-30 04:55) are each 0 commits ahead of `memory-organ`, and clean. RS-3 + and BP-3 have landed (BP-3's deferred bodies are in `huginn-daemon/src/bodies.rs`). + **Nothing is in flight.** The only other unmerged Huginn branch is + `codex/windows-atomic-replace-retry` (2026-05-24), which is unrelated. +- No Epiphany branch has unmerged commits under `epiphany-pipeline/`. +- `F:\Projects\Huginn`'s main worktree is checked out on `eureka/memory-organ`, + so the merge has to run in its own worktree. + +### B6. Deployment (read-only ssh, Yggdrasil, 2026-10-01 08:41 UTC) + +- `readlink /opt/gamecult/huginn/current` is + `releases/9dd6d8007ab7de84c926a34edd3947ba15d1f01c`. The previous release + `0c4e4107…` is kept. `huginn` is active since 2026-09-30 13:21:08 UTC. + `ExecStart` is as in the runbook (`--state-root /var/lib/gamecult/huginn + --instance eureka …`). +- `minds/` holds one mind, `eureka`: `mind.redb` 4,198,400 bytes, mtime + 2026-10-01 08:33, owned `huginn:huginn` 0640. This is the only mind on the + host. +- The nightly snapshot `/var/backups/gamecult-state/2026-10-01/` holds + `./var/lib/gamecult/huginn/minds/eureka/mind.redb` (01:59 UTC), recorded + `stable-double-read` in `consistency.tsv`. It predates receipts 81-82 (this + campaign's frame), so the deploy cut takes its own cold copy. +- Runbook: `gamecult-ops runbooks/huginn-yggdrasil.md:42-72`. Its release + section builds at a GitHub commit and keeps the previous release. It has no + pre-upgrade copy step and no read-back proof. + +## Model page + +One row per persistent kind or rule that this campaign touches. **Identity**: +what names it. **Lifecycle**: what happens to it over time. **Authority**: +who decides. + +| Kind or rule | Identity | Lifecycle | Authority | +| --- | --- | --- | --- | +| Subject keys (`finding`, `follow_up`, `question`, `ruling`, `cut_spec`, `target`, `stewardship`) | `::`, with the local bounded at 64 bytes; injective (`composed_keys_cannot_collide`, `lib.rs:1392`). **Unchanged by this campaign**: every stored key stays exactly as it is. | Admitted once; never re-keyed. A revision is a new key. | The leaf derives it (`pipeline_key`); admission refuses a mismatch (A3). Forbidden: any writer composing a key by hand. | +| Resolution keys | `:resolution:..n`, and one nesting for a withdrawal. **Target:** bounded by its own constant, derived from the deepest legal chain over the longest subject local at `u32::MAX` sequences: 10 + 1 + (11 + 1 + 64 + 2 + 10) + 2 + 10 = **111 bytes**. Ids up to 64 + 12 + 111 = 187 bytes, within `Short`. | n1, n2 … per subject; withdrawn by a depth-two resolution; never deeper (Q19 A, admission). Stored resolutions keep their keys and become withdrawable. | The leaf owns the bound and the id parser; admission owns sequence, matrix and depth. The parser bound (`pipeline_id`) is derived from the kind and is no longer `dotted_text`'s 64. | +| Every-subject-resolvable guard | **Empty by design** under Q `key-bound` option A: with subjects at 64 and resolutions at 111, every admissible subject is resolvable to depth two at any sequence, so no guard exists to own. Under option B a guard would be admission's. | | | +| Refusal classes | `MindRefusal::Document(PipelineRefusal)` for shape, bound and key; `Unavailable` only for store, cache and decode faults. | Fixed at admission time per batch. | Huginn admission. The leaf's `prepare` keeps the cache-encode error as `anyhow`. Admission keys the document first, so a key error is typed. Forbidden: `unavailable()` wrapping a leaf refusal. | +| `depends_on` | Free `Short` text today. **Target:** cut labels in the spec's own campaign (Q `depends-on`). | Stored revision-id values stay as they are (they decode, and nothing re-judges them). New batches follow the target rule. | Under A, admission refuses a new spec whose entries are not labels of cuts with a spec in the campaign. Under B, only the skill convention. | +| Sha identity | `Sha` 7-40 hex. **Target:** two shas name the same commit when one is a prefix of the other. | Only judged in a new report's batch. | A leaf method on `Sha` (the type owns its equality semantics); admission calls it at `admission.rs:433`. | +| `Deferred` referent | `PipelineRef`, any kind in the leaf. **Target:** follow_up or cut_spec, for a finding. | When the cited spec is later superseded, the reader follows `superseded_by`; nothing is rewritten. | Huginn's matrix (`admission.rs:574`). | +| Faculty | `receipt.rs` enum plus the `rows.rs:42` alias domain. **Target:** `Life` replaces `MindSteward`, with `#[serde(alias = "MindSteward")]` so any old receipt decodes. 0 stored receipts carry it (B4). | Attribution on the receipt, never re-written. | huginn-mind. The client picks it up only when rebuilt. | +| Stewardship admission rule | Rule `RepoNotStewarded` on campaign repos. | Q `stewardship-rule`. Stored stewardship documents stay; under A they decide nothing until Cut 12 (hand-off and leasing) defines a rule again. | Huginn admission; the operator owns whether the rule exists. | +| The gap home | Root `eureka-substrate`, labels `gap-*` (ruling `gap-home`). | Already moved (19 follow-ups in force here; the 20 `idunn-watchdog` copies resolved). Closed by `Fixed` when a cut lands, by Self after Soul holds. | Self admits; the skill recipe names the root (`campaign-state.md:278-299`, already done at Eureka `0940835`). No code. | +| The six stuck findings | Unchanged keys under `idunn-watchdog`. | They become resolvable when the leaf cut deploys. Self triages them then, like any finding (Deferred or Recorded). This is Self triage, not a fork. | Self. | +| Deployed release | `/opt/gamecult/huginn/releases/`, `current` symlink. | Built at a main commit, flipped in, the previous release kept, flipped back on failure. | Operator's deployment authority, exercised by Self or Hands (memory `deploy-instruction-is-authorization`). | + +Cells that could not be filled are stated in the rows above: the guard row +is empty by design if `key-bound` A is ruled, and the `depends_on` and +stewardship authorities wait on their questions. + +## Rationale + +**Why a per-kind bound and not a bigger `LOCAL_MAX`.** If one number bounds +every local, then widening it widens the subjects too, and the gap comes back +one level up: a 100-byte finding label has a 111-byte resolution. Keeping +subject locals at 64 and deriving the resolution bound from the deepest legal +chain makes resolvability a property of the grammar, with no admission guard +to drift. It also re-keys nothing: every stored key was at most 64 and stays +valid. A guard alone (keeping 64 everywhere) would refuse future offenders. +It would leave the six stuck findings and the 39 unwithdrawable resolutions +permanently broken, which violates the target for stored documents. + +**Why the parser bound moves with it.** Every reader validates ids through +`pipeline_id`. A resolution key the leaf can derive but `view`, `keys` or +`cites` refuses would turn an unresolvable subject into an unreadable +resolution. + +**Cut order.** The merges come first (ruling `merge-to-main-first`). Then the +leaf, because Huginn pins it. Then Huginn admission, which bumps the pin. Then +the client rebuild, because the new `Life` variant crosses the wire in +receipts. Then the deploy, which proves the read-back. The skill cut goes +last, because it must not tell agents to admit as `Life` before the client +and daemon understand it. In cut labels: `merge-epiphany-main`, then +`merge-huginn-main`, then `leaf-key-bound`, then `huginn-pin-and-refusals`, +then `huginn-admission-rules`, then `ops-mind-upgrade`, then `skill-recipes`. +The client rebuild is a step inside `ops-mind-upgrade`. It produces no repo +commit, and a `cut_report` needs one (`gap-fork-before-commit`). The new +daemon's added refusal variants and `Life` must not reach an old client for +long, so the client swap sits next to the daemon flip. + +**Why Cut 0 does not bump the leaf pin.** After the Epiphany merge, `73edee5c` +is an ancestor of Epiphany `main`, and the leaf tree is byte-identical. So the +daemon built from Huginn `main` is built from main code +(`code-from-main`), with an unchanged `Cargo.lock`, and the deployed release +stays as it is. The pin moves in the Huginn admission cut, to the leaf cut's +merge on Epiphany `main`. + +**Proof of `stored-documents-valid`.** Each code cut runs one read-back test +over a real copy of the mind. Every envelope decodes; `pipeline_key` of the +decoded document equals the stored key; `validate()` passes; and for every +resolvable subject the n1 resolution key and its withdrawal key derive and +parse back. The test reads the path from `HUGINN_MIND_SNAPSHOT` and is +`#[ignore]` by default, so CI never needs the file. Before code cuts, the +copy is the 2026-10-01 nightly snapshot. The deploy takes a cold copy with the +unit stopped. From 2b0cd0898b15fd9802e02a28023bf5f81483f1ea Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 11:40:28 +0200 Subject: [PATCH 046/520] huginn-mind: pin epiphany-pipeline to the per-kind key bound (leaf-key-bound) Moves the leaf from 73edee5cdaddece8bda9c6c3271e8ddd064337f9 to the leaf-key-bound merge commit on Epiphany main, ef9568655e75bb6cb362a038e2739a5ac5e777ee. Cargo.lock follows with that one line; no other lock line moves. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 2 +- crates/huginn-mind/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7254c78..25c9d81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -375,7 +375,7 @@ dependencies = [ [[package]] name = "epiphany-pipeline" version = "0.1.0" -source = "git+https://github.com/GameCult/Epiphany.git?rev=73edee5cdaddece8bda9c6c3271e8ddd064337f9#73edee5cdaddece8bda9c6c3271e8ddd064337f9" +source = "git+https://github.com/GameCult/Epiphany.git?rev=ef9568655e75bb6cb362a038e2739a5ac5e777ee#ef9568655e75bb6cb362a038e2739a5ac5e777ee" dependencies = [ "anyhow", "chrono", diff --git a/crates/huginn-mind/Cargo.toml b/crates/huginn-mind/Cargo.toml index 883e9b0..c5f3953 100644 --- a/crates/huginn-mind/Cargo.toml +++ b/crates/huginn-mind/Cargo.toml @@ -19,7 +19,7 @@ base64 = "0.22" chrono = "0.4.44" cultcache-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultcache-rs" } cultnet-rs = { git = "https://github.com/GameCult/CultLib.git", rev = "8fc74c709444b884a2bd339e0c306bf9484d99d2", package = "cultnet-rs" } -epiphany-pipeline = { git = "https://github.com/GameCult/Epiphany.git", rev = "73edee5cdaddece8bda9c6c3271e8ddd064337f9", package = "epiphany-pipeline" } +epiphany-pipeline = { git = "https://github.com/GameCult/Epiphany.git", rev = "ef9568655e75bb6cb362a038e2739a5ac5e777ee", package = "epiphany-pipeline" } rmp-serde = "1" schemars = "1" serde = { version = "1", features = ["derive"] } From 790b6cd4f68de70a37f9a537b5dc44de4883d27b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 11:40:29 +0200 Subject: [PATCH 047/520] huginn-mind: refuse an unkeyable document as a document, and prove the real mind reads back Admission wrapped every prepare error in Unavailable, so a key the leaf could not compose reached the client as an unavailable organ. One private prepare(document, cache) now keys through the leaf first, mapped to MindRefusal::Document, then encodes; Mind::admit and the derived-write loop both use it. Unavailable stays for store, cache-encode and decode faults. Tests: an_unkeyable_document_is_refused_as_a_document (store and receipts unchanged), prepare_classifies_a_key_refusal_as_a_document_and_encodes_the_rest (the derived path cannot reach an unkeyable resolution after the per-kind bound, so the helper is pinned directly), and the #[ignore] snapshot_admits_and_reads_back, which opens a scratch copy of the live mind named by HUGINN_MIND_SNAPSHOT, views every stored document back unchanged, resolves the six stuck findings and withdraws one long-keyed resolution. Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/admission.rs | 65 +++++++++++++++++++++++++-- crates/huginn-mind/src/mind.rs | 68 +++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 4 deletions(-) diff --git a/crates/huginn-mind/src/admission.rs b/crates/huginn-mind/src/admission.rs index 6443e94..487465d 100644 --- a/crates/huginn-mind/src/admission.rs +++ b/crates/huginn-mind/src/admission.rs @@ -29,7 +29,7 @@ use std::collections::BTreeSet; use chrono::{DateTime, Utc}; -use cultcache_rs::CultCacheEnvelope; +use cultcache_rs::{CultCache, CultCacheEnvelope}; use epiphany_pipeline::{ ClaimOutcome, FindingConfidence, Line, PipelineDocument, PipelineKind, PipelineRef, PipelineRefusal, PipelineResolution, PipelineStewardship, ResolutionOutcome, RulingAuthority, Short, Slug, pipeline_key, @@ -67,6 +67,14 @@ pub enum PipelineAdmissionOutcome { Conflict { identities: Vec }, } +/// Keys a document through the leaf, then encodes it. A key the leaf cannot +/// compose is the document's fault and is refused as a document; only the +/// cache's encode fault is the organ's. +fn prepare(document: &PipelineDocument, cache: &CultCache) -> Result { + pipeline_key(document).map_err(MindRefusal::Document)?; + document.prepare(cache).map_err(unavailable) +} + impl Mind { /// Validates and prepares every document through the leaf, then admits /// the envelopes. @@ -76,9 +84,9 @@ impl Mind { if let Err(refusal) = document.validate() { return PipelineAdmissionOutcome::Refused(MindRefusal::Document(refusal)); } - match document.prepare(self.cache()) { + match prepare(document, self.cache()) { Ok(envelope) => envelopes.push(envelope), - Err(error) => return PipelineAdmissionOutcome::Refused(unavailable(error)), + Err(refusal) => return PipelineAdmissionOutcome::Refused(refusal), } } self.admit_prepared(&batch.instance, batch.provenance, envelopes, now) @@ -133,7 +141,7 @@ impl Mind { let originals = docs.batch.len(); for document in derive(&docs, &mind) { document.validate()?; - let envelope = document.prepare(self.cache()).map_err(unavailable)?; + let envelope = prepare(&document, self.cache())?; docs.push(stage(envelope, &mind)?)?; } for staged in &docs.batch[originals..] { @@ -684,6 +692,8 @@ fn refuse_collisions(docs: &Docs) -> Result<(), MindRefusal> { mod tests { use super::*; use crate::fixtures::*; + use crate::fixtures::prepare; + use crate::mind::schema_cache; use crate::receipt::{DocumentVersion, Faculty, HuginnCommitReceipt}; use crate::store::test_stores::{MemoryStore, RefusingStore, SwapCommand}; use cultcache_rs::DatabaseEntry; @@ -1671,6 +1681,53 @@ mod tests { ); } + /// refusals-typed: a document the leaf cannot key is the client's fault. + /// The finding's local is 64 bytes of label under `cut-1.s1.`; the + /// resolution's subject is a resolution whose own local leaves no room for + /// a third nesting. Both pass `validate` and fail only at the key. + fn unkeyable() -> Vec { + let wide = D::Finding(finding("1", 1, &"w".repeat(64), FindingConfidence::Confirmed)); + let deep_subject = id("resolution", &format!("{}.{}", "x".repeat(60), "y".repeat(40))); + vec![wide, resolution(r(K::Resolution, &deep_subject), withdrawn())] + } + + #[test] + fn an_unkeyable_document_is_refused_as_a_document() { + let store = RefusingStore::new(); + let mut mind = opened(store.clone(), INSTANCE); + seed(&mut mind); + let before = store.rows(); + let receipts = receipt_count(&mind); + for document in unkeyable() { + document.validate().unwrap(); + match refusal(admit(&mut mind, vec![document])) { + MindRefusal::Document(PipelineRefusal::InvalidFormat { field, .. }) => { + assert!(field.ends_with(".key"), "{field}") + } + other => panic!("{other:?}"), + } + assert_eq!(store.rows(), before); + assert_eq!(receipt_count(&mind), receipts); + } + } + + /// The derived path (`admit_steps`) shares the helper. No derivable + /// document can fail to key after the leaf's per-kind bound (a derived + /// resolution is depth one or two over a subject of at most 64 bytes), so + /// the helper is pinned directly. + #[test] + fn prepare_classifies_a_key_refusal_as_a_document_and_encodes_the_rest() { + let cache = schema_cache().unwrap(); + let keyable = question("Q1", &["A", "B"], "A"); + assert_eq!(super::prepare(&keyable, &cache).unwrap(), prepare(&keyable)); + for document in unkeyable() { + assert!(matches!( + super::prepare(&document, &cache), + Err(MindRefusal::Document(PipelineRefusal::InvalidFormat { .. })) + )); + } + } + #[test] fn a_ruling_answering_a_question_derives_the_answered_resolution_atomically() { let mut mind = seeded(); diff --git a/crates/huginn-mind/src/mind.rs b/crates/huginn-mind/src/mind.rs index ff5fbc2..f89a1af 100644 --- a/crates/huginn-mind/src/mind.rs +++ b/crates/huginn-mind/src/mind.rs @@ -355,6 +355,7 @@ fn snapshot(cache: &CultCache) -> Vec { #[cfg(test)] mod tests { use super::*; + use crate::admission::PipelineAdmissionOutcome; use crate::fixtures::{INSTANCE, epoch, instance, prepare, slug}; use crate::store::test_stores::MemoryStore; use cultcache_rs::CacheBackingStore; @@ -778,4 +779,71 @@ mod tests { })) ); } + + /// stored-documents-valid and every-subject-resolvable, against the real + /// mind: every stored document views back unchanged; the six findings the + /// 64-byte bound left unresolvable take a resolution; and a stored + /// resolution whose withdrawal key is over 64 bytes is withdrawn. Run it + /// on a scratch copy of the state root only: it writes. + #[test] + #[ignore = "reads and writes the scratch copy of a live state root named by HUGINN_MIND_SNAPSHOT"] + fn snapshot_admits_and_reads_back() { + use crate::fixtures::{admit, committed, r, resolution, withdrawn}; + use epiphany_pipeline::{PipelineRef, ResolutionOutcome, Short, pipeline_key}; + + let root = std::env::var("HUGINN_MIND_SNAPSHOT").expect("HUGINN_MIND_SNAPSHOT names a scratch copy of the state root"); + let mut mind = Mind::open(Path::new(&root), &slug("eureka")).unwrap(); + + let mut viewed = 0; + let mut resolutions = Vec::new(); + for envelope in mind.envelopes() { + let Some(kind) = PipelineKind::ALL.iter().copied().find(|kind| kind.type_id() == envelope.r#type) else { continue }; + let stored = PipelineDocument::decode(envelope).unwrap(); + let view = mind + .view(&PipelineRef { kind, id: Short(envelope.key.clone()) }) + .unwrap() + .unwrap_or_else(|| panic!("{} does not view", envelope.key)); + assert_eq!(view.document, stored, "{}", envelope.key); + if let PipelineDocument::Resolution(held) = &stored + && !matches!(held.outcome, epiphany_pipeline::ResolutionOutcome::Withdrawn { .. }) + && !envelope.key.contains(":resolution:resolution.") + { + resolutions.push(envelope.key.clone()); + } + viewed += 1; + } + assert!(viewed > 0, "the snapshot held no pipeline document"); + println!("viewed {viewed} stored documents unchanged"); + + for local in [ + "cut-bifrost-retire-alarm.s2.verb-default-accepts-malformed", + "cut-ops-notice-deploy.s2.pinned-bifrost-predates-retry", + "cut-bifrost-notice-retry.s1.unknown-test-ignores-backoff", + "cut-bifrost-notice-retry.s2.closed-unknown-retry-unpinned", + "cut-bifrost-notice-retry.s2.flapping-clock-retries-every-tick", + "cut-idunn-topology-lock.s2.boot-reconcile-skipped-on-contention", + ] { + let subject = r(PipelineKind::Finding, &format!("idunn-watchdog:finding:{local}")); + let outcome = admit(&mut mind, vec![resolution(subject, ResolutionOutcome::Recorded { reason: "snapshot read-back".into() })]); + let (_, writes) = committed(outcome); + assert_eq!(writes.len(), 1, "{local}"); + } + + let mut tried = 0; + let mut withdrew = None; + for key in resolutions { + let withdrawal = resolution(r(PipelineKind::Resolution, &key), withdrawn()); + let withdrawal_key = pipeline_key(&withdrawal).unwrap(); + if withdrawal_key.rsplit(':').next().unwrap().len() <= 64 { + continue; + } + tried += 1; + if let PipelineAdmissionOutcome::Committed { .. } = admit(&mut mind, vec![withdrawal]) { + withdrew = Some(withdrawal_key); + break; + } + } + let withdrew = withdrew.unwrap_or_else(|| panic!("no stored resolution with a withdrawal key over 64 bytes withdrew; tried {tried}")); + println!("withdrew a resolution under a {}-byte key", withdrew.len()); + } } From f29a7ce7f0bc3977e2adb625cb09ec6b28646d4b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 12:10:19 +0200 Subject: [PATCH 048/520] huginn-mind: tighten the refusal and read-back tests (Soul s1 findings) snapshot_admits_and_reads_back now counts pipeline envelopes apart from the loop, asserts validate() and pipeline_key == stored key for every one, admits a resolution for every stuck finding and a withdrawal for every unwithdrawable resolution, and fails with the id named on any refusal. The refusal tests assert the exact leaf refusal (field and value from pipeline_key) at Mind::admit and at the helper, and eureka-state gains a tool-level test that the typed refusal reaches the result intact. Tests only; no production code changes. Co-Authored-By: Claude Sonnet 5.5 --- crates/eureka-state/tests/mcp.rs | 34 +++++++++++++++- crates/huginn-mind/src/admission.rs | 22 +++++------ crates/huginn-mind/src/mind.rs | 60 ++++++++++++++++++++--------- 3 files changed, 85 insertions(+), 31 deletions(-) diff --git a/crates/eureka-state/tests/mcp.rs b/crates/eureka-state/tests/mcp.rs index 27ee9fd..969ad55 100644 --- a/crates/eureka-state/tests/mcp.rs +++ b/crates/eureka-state/tests/mcp.rs @@ -11,7 +11,7 @@ use std::time::Duration; use huginn_mind::envelope::{OperationFailure, encode_failure, encode_response}; use huginn_mind::{HuginnMindRequest, HuginnMindResponse}; -use huginn_mind::epiphany_pipeline::{PipelineKind, PipelineRef, Short}; +use huginn_mind::epiphany_pipeline::{Date, PipelineDocument, PipelineKind, PipelineRef, Short}; use serde_json::{Value, json}; mod common; @@ -530,3 +530,35 @@ fn initialize_advertises_tools() { let mcp = Mcp::at(INSTANCE, closed_port()); assert!(mcp.init["capabilities"]["tools"].is_object(), "{}", mcp.init); } + +/// A document the leaf cannot key is the client's fault, and the tool result +/// carries the leaf's own refusal: the same field and the same offending +/// value `pipeline_key` computes for that document, never a re-built one. +#[test] +fn admit_carries_the_leaf_key_refusal_with_its_field_and_value() { + use huginn_mind::epiphany_pipeline::{PipelineResolution, ResolutionOutcome, pipeline_key}; + + let (root, daemon) = mind(vec![vec![stewardship()]]); + let server = serve(root, daemon); + let mut mcp = Mcp::at(INSTANCE, server.addr); + // A resolution of a resolution of a resolution: it validates and has no + // room left in the key for a third nesting. + let subject = format!("{CAMPAIGN}:resolution:{}.{}", "x".repeat(60), "y".repeat(40)); + let document = PipelineDocument::Resolution(PipelineResolution { + subject: PipelineRef { kind: PipelineKind::Resolution, id: Short(subject) }, + sequence: 1, + outcome: ResolutionOutcome::Withdrawn { reason: "moot".into() }, + rationale: "Resolved.".into(), + resolved_on: Date("2026-09-29".into()), + }); + document.validate().unwrap(); + let leaf = pipeline_key(&document).expect_err("the fixture must not key"); + let expected = serde_json::to_value(&leaf).unwrap(); + assert_eq!(expected["InvalidFormat"]["field"], json!("resolution.key")); + + let (error, refused) = mcp.call("admit", faculty_args(vec![serde_json::to_value(&document).unwrap()])); + assert!(!error, "{refused}"); + assert_eq!(refused["Refused"]["Document"], expected, "{refused}"); + let (_, whoami) = mcp.call("whoami", json!({})); + assert_eq!(whoami["status"]["receipts"], json!(2), "only the seed is committed: {whoami}"); +} diff --git a/crates/huginn-mind/src/admission.rs b/crates/huginn-mind/src/admission.rs index 487465d..f2baeea 100644 --- a/crates/huginn-mind/src/admission.rs +++ b/crates/huginn-mind/src/admission.rs @@ -1698,14 +1698,16 @@ mod tests { seed(&mut mind); let before = store.rows(); let receipts = receipt_count(&mind); - for document in unkeyable() { + for (document, field) in unkeyable().into_iter().zip(["finding.key", "resolution.key"]) { document.validate().unwrap(); - match refusal(admit(&mut mind, vec![document])) { - MindRefusal::Document(PipelineRefusal::InvalidFormat { field, .. }) => { - assert!(field.ends_with(".key"), "{field}") - } - other => panic!("{other:?}"), - } + // The leaf owns what a key error says; admission must carry it + // unchanged, field and offending value included. + let leaf = pipeline_key(&document).expect_err("the fixture must not key"); + assert!( + matches!(&leaf, PipelineRefusal::InvalidFormat { field: named, value } if named == field && !value.is_empty()), + "{leaf:?}" + ); + assert_eq!(refusal(admit(&mut mind, vec![document])), MindRefusal::Document(leaf), "{field}"); assert_eq!(store.rows(), before); assert_eq!(receipt_count(&mind), receipts); } @@ -1721,10 +1723,8 @@ mod tests { let keyable = question("Q1", &["A", "B"], "A"); assert_eq!(super::prepare(&keyable, &cache).unwrap(), prepare(&keyable)); for document in unkeyable() { - assert!(matches!( - super::prepare(&document, &cache), - Err(MindRefusal::Document(PipelineRefusal::InvalidFormat { .. })) - )); + let leaf = pipeline_key(&document).expect_err("the fixture must not key"); + assert_eq!(super::prepare(&document, &cache).unwrap_err(), MindRefusal::Document(leaf)); } } diff --git a/crates/huginn-mind/src/mind.rs b/crates/huginn-mind/src/mind.rs index f89a1af..4287f5f 100644 --- a/crates/huginn-mind/src/mind.rs +++ b/crates/huginn-mind/src/mind.rs @@ -788,17 +788,27 @@ mod tests { #[test] #[ignore = "reads and writes the scratch copy of a live state root named by HUGINN_MIND_SNAPSHOT"] fn snapshot_admits_and_reads_back() { - use crate::fixtures::{admit, committed, r, resolution, withdrawn}; + use crate::fixtures::{admit, r, resolution, withdrawn}; use epiphany_pipeline::{PipelineRef, ResolutionOutcome, Short, pipeline_key}; let root = std::env::var("HUGINN_MIND_SNAPSHOT").expect("HUGINN_MIND_SNAPSHOT names a scratch copy of the state root"); let mut mind = Mind::open(Path::new(&root), &slug("eureka")).unwrap(); + // The census is taken apart from the loop that checks each document. + let pipeline_envelopes = mind + .envelopes() + .filter(|envelope| PipelineKind::ALL.iter().any(|kind| kind.type_id() == envelope.r#type)) + .count(); + assert!(pipeline_envelopes > 0, "the snapshot held no pipeline document"); + let mut viewed = 0; let mut resolutions = Vec::new(); for envelope in mind.envelopes() { let Some(kind) = PipelineKind::ALL.iter().copied().find(|kind| kind.type_id() == envelope.r#type) else { continue }; let stored = PipelineDocument::decode(envelope).unwrap(); + stored.validate().unwrap_or_else(|refusal| panic!("{} no longer validates: {refusal:?}", envelope.key)); + let rekeyed = pipeline_key(&stored).unwrap_or_else(|refusal| panic!("{} no longer keys: {refusal:?}", envelope.key)); + assert_eq!(rekeyed, envelope.key, "stored key differs from the key the leaf computes"); let view = mind .view(&PipelineRef { kind, id: Short(envelope.key.clone()) }) .unwrap() @@ -812,38 +822,50 @@ mod tests { } viewed += 1; } - assert!(viewed > 0, "the snapshot held no pipeline document"); - println!("viewed {viewed} stored documents unchanged"); + assert_eq!(viewed, pipeline_envelopes, "every pipeline envelope validates, rekeys and views back"); + println!("validated, rekeyed and viewed {viewed} stored documents unchanged"); - for local in [ + let stuck = [ "cut-bifrost-retire-alarm.s2.verb-default-accepts-malformed", "cut-ops-notice-deploy.s2.pinned-bifrost-predates-retry", "cut-bifrost-notice-retry.s1.unknown-test-ignores-backoff", "cut-bifrost-notice-retry.s2.closed-unknown-retry-unpinned", "cut-bifrost-notice-retry.s2.flapping-clock-retries-every-tick", "cut-idunn-topology-lock.s2.boot-reconcile-skipped-on-contention", - ] { + ]; + let mut resolved = 0; + for local in stuck { let subject = r(PipelineKind::Finding, &format!("idunn-watchdog:finding:{local}")); let outcome = admit(&mut mind, vec![resolution(subject, ResolutionOutcome::Recorded { reason: "snapshot read-back".into() })]); - let (_, writes) = committed(outcome); - assert_eq!(writes.len(), 1, "{local}"); + match outcome { + PipelineAdmissionOutcome::Committed { writes, .. } => { + assert_eq!(writes.len(), 1, "{local}"); + resolved += 1; + } + other => panic!("stuck finding {local} was not resolved: {other:?}"), + } } - let mut tried = 0; - let mut withdrew = None; + assert_eq!(resolved, stuck.len(), "every stuck finding resolves"); + + let mut candidates = Vec::new(); for key in resolutions { let withdrawal = resolution(r(PipelineKind::Resolution, &key), withdrawn()); - let withdrawal_key = pipeline_key(&withdrawal).unwrap(); - if withdrawal_key.rsplit(':').next().unwrap().len() <= 64 { - continue; - } - tried += 1; - if let PipelineAdmissionOutcome::Committed { .. } = admit(&mut mind, vec![withdrawal]) { - withdrew = Some(withdrawal_key); - break; + let withdrawal_key = pipeline_key(&withdrawal).unwrap_or_else(|refusal| panic!("withdrawal of {key} does not key: {refusal:?}")); + if withdrawal_key.rsplit(':').next().unwrap().len() > 64 { + candidates.push((key, withdrawal)); } } - let withdrew = withdrew.unwrap_or_else(|| panic!("no stored resolution with a withdrawal key over 64 bytes withdrew; tried {tried}")); - println!("withdrew a resolution under a {}-byte key", withdrew.len()); + assert!(!candidates.is_empty(), "no stored resolution had a withdrawal key over 64 bytes"); + let wanted = candidates.len(); + let mut withdrew = 0; + for (key, withdrawal) in candidates { + match admit(&mut mind, vec![withdrawal]) { + PipelineAdmissionOutcome::Committed { .. } => withdrew += 1, + other => panic!("withdrawal of {key} was not committed: {other:?}"), + } + } + assert_eq!(withdrew, wanted, "every candidate withdraws"); + println!("withdrew {withdrew} resolutions whose withdrawal keys exceed 64 bytes"); } } From 0f491b00ef971b49f024d4341c8c309065871772 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 12:12:36 +0200 Subject: [PATCH 049/520] huginn-mind: iterate the envelope slice in the read-back census Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/mind.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/huginn-mind/src/mind.rs b/crates/huginn-mind/src/mind.rs index 4287f5f..619471e 100644 --- a/crates/huginn-mind/src/mind.rs +++ b/crates/huginn-mind/src/mind.rs @@ -797,6 +797,7 @@ mod tests { // The census is taken apart from the loop that checks each document. let pipeline_envelopes = mind .envelopes() + .iter() .filter(|envelope| PipelineKind::ALL.iter().any(|kind| kind.type_id() == envelope.r#type)) .count(); assert!(pipeline_envelopes > 0, "the snapshot held no pipeline document"); From db8430c6fe580e52d0c135436cef6a7eba11de85 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 12:50:38 +0200 Subject: [PATCH 050/520] huginn-mind: report heads match by commit, findings defer to cut specs, MindSteward is Life Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/admission.rs | 47 +++++++++++++++++++++++++++-- crates/huginn-mind/src/query.rs | 3 ++ crates/huginn-mind/src/receipt.rs | 14 ++++++++- crates/huginn-mind/src/rows.rs | 2 +- 4 files changed, 62 insertions(+), 4 deletions(-) diff --git a/crates/huginn-mind/src/admission.rs b/crates/huginn-mind/src/admission.rs index f2baeea..4b952cf 100644 --- a/crates/huginn-mind/src/admission.rs +++ b/crates/huginn-mind/src/admission.rs @@ -438,7 +438,7 @@ fn check(docs: &Docs, staged: &Staged, mind: &Slug) -> Result<(), MindRefusal> { if report.branch != spec.branch { return Err(MindRefusal::SpecMismatch { field: "branch".into() }); } - if !report.commits.iter().any(|commit| commit.sha == report.range.head) { + if !report.commits.iter().any(|commit| commit.sha.names_same_commit(&report.range.head)) { return Err(MindRefusal::RangeOutsideCommits { head: report.range.head.0.clone() }); } Ok(()) @@ -579,7 +579,7 @@ fn matrix(subject: PipelineKind, outcome: &ResolutionOutcome) -> bool { (K::CutSpec, O::Superseded { by }) => all(by, K::CutSpec), (K::CutSpec, O::Withdrawn { .. }) => true, (K::Finding, O::Fixed { by, .. }) => by.as_ref().is_none_or(|report| report.kind == K::CutReport), - (K::Finding, O::Deferred { to }) => to.kind == K::FollowUp, + (K::Finding, O::Deferred { to }) => matches!(to.kind, K::FollowUp | K::CutSpec), (K::Finding, O::Recorded { .. } | O::Withdrawn { .. }) => true, (K::FollowUp, O::Fixed { by, .. }) => by.as_ref().is_none_or(|report| report.kind == K::CutReport), (K::FollowUp, O::Superseded { by }) => all(by, K::FollowUp), @@ -1910,6 +1910,49 @@ mod tests { ); } + #[test] + fn a_report_head_may_spell_its_commit_short_or_full() { + let full = "5f98228d0123456789abcdef0123456789abcdef"; + let mut mind = seeded(); + committed(admit(&mut mind, vec![D::CutSpec(cut_spec("1", 1))])); + // The commit is spelled short and the head full, and the reverse. + let mut report = cut_report("1", 1); + report.range.head = epiphany_pipeline::Sha(full.into()); + committed(admit(&mut mind, vec![D::CutReport(report)])); + let mut report = cut_report("1", 2); + report.commits[0].sha = epiphany_pipeline::Sha(full.into()); + committed(admit(&mut mind, vec![D::CutReport(report)])); + // A head that is no commit's prefix, long or short, is outside. + let mut report = cut_report("1", 3); + report.range.head = epiphany_pipeline::Sha("5f98229d0123456789abcdef0123456789abcdef".into()); + assert!(matches!(refusal(admit(&mut mind, vec![D::CutReport(report)])), MindRefusal::RangeOutsideCommits { .. })); + let mut report = cut_report("1", 3); + report.commits[0].sha = epiphany_pipeline::Sha(full.into()); + report.range.head = epiphany_pipeline::Sha("5f98229".into()); + assert!(matches!(refusal(admit(&mut mind, vec![D::CutReport(report)])), MindRefusal::RangeOutsideCommits { .. })); + } + + #[test] + fn a_finding_defers_to_a_cut_spec() { + let finding_ref = r(K::Finding, &id("finding", "cut-1.s1.F1")); + let defer_to = |to: PipelineRef| resolution(finding_ref.clone(), ResolutionOutcome::Deferred { to }); + committed(admit(&mut world(), vec![defer_to(r(K::CutSpec, &id("cut_spec", "cut-1.r1")))])); + // A spec superseded since is no longer a place to defer to. + let mut mind = world(); + committed(admit(&mut mind, vec![ + D::CutSpec(cut_spec("1", 2)), + resolution(r(K::CutSpec, &id("cut_spec", "cut-1.r1")), superseded(&[r(K::CutSpec, &id("cut_spec", "cut-1.r2"))])), + ])); + assert_eq!( + refusal(admit(&mut mind, vec![defer_to(r(K::CutSpec, &id("cut_spec", "cut-1.r1")))])), + MindRefusal::CitesResolvedDocument { kind: K::CutSpec, id: id("cut_spec", "cut-1.r1") } + ); + assert_eq!( + refusal(admit(&mut world(), vec![defer_to(r(K::Ruling, &id("ruling", "R1")))])), + MindRefusal::IncompatibleResolution { subject_kind: K::Finding, outcome: "Deferred".into() } + ); + } + #[test] fn verdict_vocabulary_binds_claims_to_findings_promises_and_mutations() { let mut mind = seeded(); diff --git a/crates/huginn-mind/src/query.rs b/crates/huginn-mind/src/query.rs index 274d902..16a1d95 100644 --- a/crates/huginn-mind/src/query.rs +++ b/crates/huginn-mind/src/query.rs @@ -1046,6 +1046,9 @@ mod tests { // `faculty`: attribution selects, and grants nothing (ruling 18). assert_eq!(by("faculty", &["Soul"]), vec![id("ruling", "R2")]); assert!(!by("faculty", &["Hands"]).contains(&id("ruling", "R2"))); + // The renamed faculty is selectable by its new name only. + assert!(by("faculty", &["Life"]).is_empty()); + assert!(invalid("fields[0].values", "MindSteward")(&refused(&mind, &with(Selection::default(), any_of("faculty", &["MindSteward"]))))); // A repo is named by its identity, which is case-insensitive. assert_eq!(by("repo", &["GAMECULT/EPIPHANY"]), epiphany); diff --git a/crates/huginn-mind/src/receipt.rs b/crates/huginn-mind/src/receipt.rs index f08188d..3abbf16 100644 --- a/crates/huginn-mind/src/receipt.rs +++ b/crates/huginn-mind/src/receipt.rs @@ -36,7 +36,8 @@ pub enum Faculty { Imagination, Hands, Soul, - MindSteward, + #[serde(alias = "MindSteward")] + Life, Eyes, Operator, } @@ -295,6 +296,17 @@ mod tests { use std::sync::{Arc, Mutex}; use super::*; + + /// A receipt written before the rename spells the faculty `MindSteward`; + /// it decodes as `Life`, and `Life` is what a new one writes. + #[test] + fn life_is_the_faculty_and_the_old_spelling_decodes() { + let old = rmp_serde::to_vec_named("MindSteward").unwrap(); + assert_eq!(rmp_serde::from_slice::(&old).unwrap(), Faculty::Life); + let new = rmp_serde::to_vec_named(&Faculty::Life).unwrap(); + assert_eq!(rmp_serde::from_slice::(&new).unwrap(), "Life"); + } + use crate::fixtures::{INSTANCE, admit, committed, id, instance, now, opened, prepare, provenance, question, refusal, seed, slug}; use crate::mind::schema_cache; use crate::store::test_stores::MemoryStore; diff --git a/crates/huginn-mind/src/rows.rs b/crates/huginn-mind/src/rows.rs index d25f53c..108cc9c 100644 --- a/crates/huginn-mind/src/rows.rs +++ b/crates/huginn-mind/src/rows.rs @@ -39,7 +39,7 @@ pub(crate) enum Alias { Outcome, } -const FACULTIES: [&str; 7] = ["SelfFaculty", "Imagination", "Hands", "Soul", "MindSteward", "Eyes", "Operator"]; +const FACULTIES: [&str; 7] = ["SelfFaculty", "Imagination", "Hands", "Soul", "Life", "Eyes", "Operator"]; const SEVERITIES: [&str; 4] = ["Blocker", "High", "Medium", "Low"]; const CONFIDENCES: [&str; 2] = ["Confirmed", "Plausible"]; const ORIGINS: [&str; 2] = ["Introduced", "PreExisting"]; From 08d48957f8e5244d85b50d976a93839c39a879ba Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 12:50:57 +0200 Subject: [PATCH 051/520] huginn-mind: campaign admission no longer checks repo stewardship Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/admission.rs | 30 +++++------------------------ crates/huginn-mind/src/refusal.rs | 1 - 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/crates/huginn-mind/src/admission.rs b/crates/huginn-mind/src/admission.rs index 4b952cf..55daf34 100644 --- a/crates/huginn-mind/src/admission.rs +++ b/crates/huginn-mind/src/admission.rs @@ -368,11 +368,6 @@ fn check(docs: &Docs, staged: &Staged, mind: &Slug) -> Result<(), MindRefusal> { if campaign.repos.is_empty() { return Err(MindRefusal::EmptyRepos { campaign: key.into() }); } - for repo in &campaign.repos { - if docs.stewardship_of(mind, repo, None).is_none() { - return Err(MindRefusal::RepoNotStewarded { repo: repo.0.clone() }); - } - } Ok(()) } D::Target(target) => { @@ -1350,14 +1345,11 @@ mod tests { receipt.strong_reads ); - // Nothing stewards the repo now, and `stewardship_of` says so: a new - // campaign over it is refused. + // Nothing stewards the repo now, and a campaign over it still admits: + // campaign admission reads no stewardship (ruling stewardship-rule). let D::Campaign(mut second) = campaign(&[REPO]) else { panic!() }; second.slug = slug("second"); - assert_eq!( - refusal(admit(&mut mind, vec![D::Campaign(second)])), - MindRefusal::RepoNotStewarded { repo: REPO.into() } - ); + committed(admit(&mut mind, vec![D::Campaign(second)])); // Handed back, the mind stewards it again, as `n2`. No field names a // return: this is the same hand-off shape with the instances swapped. @@ -2084,27 +2076,15 @@ mod tests { } #[test] - fn a_campaign_names_only_repos_this_mind_stewards() { + fn a_campaign_needs_no_stewardship() { let mut mind = opened(MemoryStore::new(), INSTANCE); - assert_eq!( - refusal(admit(&mut mind, vec![instance(INSTANCE), campaign(&[REPO])])), - MindRefusal::RepoNotStewarded { repo: REPO.into() } - ); - committed(admit(&mut mind, vec![instance(INSTANCE), stewardship(INSTANCE, REPO), campaign(&[REPO])])); - // A campaign of no repos is the organ's refusal, not a leaf bound: - // "every repo is stewarded" is vacuous and the campaign steers - // nothing. + committed(admit(&mut mind, vec![instance(INSTANCE), campaign(&[REPO])])); let D::Campaign(mut empty) = campaign(&[]) else { panic!() }; empty.slug = slug("other"); assert_eq!( refusal(admit(&mut mind, vec![D::Campaign(empty)])), MindRefusal::EmptyRepos { campaign: "other:campaign:self".into() } ); - // The stewardship may be held in the image: a later campaign over the - // same repo carries no stewardship of its own. - let D::Campaign(mut second) = campaign(&[REPO]) else { panic!() }; - second.slug = slug("second"); - committed(admit(&mut mind, vec![D::Campaign(second)])); let mut spec = cut_spec("1", 1); spec.repo = repo(OTHER_REPO); assert_eq!(refusal(admit(&mut mind, vec![D::CutSpec(spec)])), MindRefusal::RepoNotInCampaign { repo: OTHER_REPO.into() }); diff --git a/crates/huginn-mind/src/refusal.rs b/crates/huginn-mind/src/refusal.rs index ae369d0..8e0e058 100644 --- a/crates/huginn-mind/src/refusal.rs +++ b/crates/huginn-mind/src/refusal.rs @@ -42,7 +42,6 @@ pub enum MindRefusal { InvalidChoice { ruling: String, choice: String }, QuoteWithoutOperator { ruling: String }, EmptyRepos { campaign: String }, - RepoNotStewarded { repo: String }, RepoNotInCampaign { repo: String }, CutReportWithoutSpec { report: String }, SpecMismatch { field: String }, From cfa8d570ffc2c2a717796791b74fb6b821c33ee6 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 12:54:20 +0200 Subject: [PATCH 052/520] huginn-mind: a cut spec's depends_on must name cuts of its campaign Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/admission.rs | 43 +++++++++++++++++++++++++++++ crates/huginn-mind/src/refusal.rs | 1 + 2 files changed, 44 insertions(+) diff --git a/crates/huginn-mind/src/admission.rs b/crates/huginn-mind/src/admission.rs index 55daf34..3981bb5 100644 --- a/crates/huginn-mind/src/admission.rs +++ b/crates/huginn-mind/src/admission.rs @@ -417,6 +417,14 @@ fn check(docs: &Docs, staged: &Staged, mind: &Slug) -> Result<(), MindRefusal> { return Err(MindRefusal::CitesResolvedDocument { kind: K::Ruling, id: ruling.0.clone() }); } } + for dependency in &spec.depends_on { + let names_a_cut = docs.of_kind(K::CutSpec).any(|(_, document)| { + matches!(document, D::CutSpec(other) if other.campaign == spec.campaign && other.cut.0 == dependency.0) + }); + if !names_a_cut { + return Err(MindRefusal::UnknownDependency { cut: dependency.0.clone() }); + } + } let predecessor = D::CutSpec(epiphany_pipeline::PipelineCutSpec { revision: spec.revision.wrapping_sub(1), ..spec.clone() }); revision_rule(docs, K::CutSpec, key, spec.revision, &predecessor) } @@ -1902,6 +1910,41 @@ mod tests { ); } + #[test] + fn depends_on_names_cuts() { + let depending = |cut: &str, on: &[&str]| { + let mut spec = cut_spec(cut, 1); + spec.depends_on = on.iter().map(|label| s(label)).collect(); + D::CutSpec(spec) + }; + let mut mind = seeded(); + // An unknown label, and a spec id (a revision, not a cut), are refused. + assert_eq!( + refusal(admit(&mut mind, vec![depending("2", &["1"])])), + MindRefusal::UnknownDependency { cut: "1".into() } + ); + assert_eq!( + refusal(admit(&mut mind, vec![depending("2", &[&id("cut_spec", "cut-1.r1")])])), + MindRefusal::UnknownDependency { cut: id("cut_spec", "cut-1.r1") } + ); + // The same batch carries the cut it depends on. + committed(admit(&mut mind, vec![D::CutSpec(cut_spec("1", 1)), depending("2", &["1"])])); + // The image carries it, and any revision of the cut names it. + committed(admit(&mut mind, vec![depending("3", &["1", "2"])])); + // A cut of another campaign is not this campaign's cut. + let mut other = cut_spec("9", 1); + other.campaign = slug("elsewhere"); + let mut elsewhere = campaign(&[REPO]); + if let D::Campaign(campaign) = &mut elsewhere { + campaign.slug = slug("elsewhere"); + } + committed(admit(&mut mind, vec![elsewhere, D::CutSpec(other)])); + assert_eq!( + refusal(admit(&mut mind, vec![depending("4", &["9"])])), + MindRefusal::UnknownDependency { cut: "9".into() } + ); + } + #[test] fn a_report_head_may_spell_its_commit_short_or_full() { let full = "5f98228d0123456789abcdef0123456789abcdef"; diff --git a/crates/huginn-mind/src/refusal.rs b/crates/huginn-mind/src/refusal.rs index 8e0e058..1a33ac7 100644 --- a/crates/huginn-mind/src/refusal.rs +++ b/crates/huginn-mind/src/refusal.rs @@ -43,6 +43,7 @@ pub enum MindRefusal { QuoteWithoutOperator { ruling: String }, EmptyRepos { campaign: String }, RepoNotInCampaign { repo: String }, + UnknownDependency { cut: String }, CutReportWithoutSpec { report: String }, SpecMismatch { field: String }, RangeOutsideCommits { head: String }, From 56baa46b116532627e4ec13c96c70762f1d1207d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 12:57:55 +0200 Subject: [PATCH 053/520] huginn-mind: published wire schemas follow the renames; the summary bound test uses real cuts Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/query.rs | 11 ++++- .../huginn.mind_request.v1.schema.json | 2 +- .../huginn.mind_response.v1.schema.json | 42 +++++++++---------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/crates/huginn-mind/src/query.rs b/crates/huginn-mind/src/query.rs index 16a1d95..9d10009 100644 --- a/crates/huginn-mind/src/query.rs +++ b/crates/huginn-mind/src/query.rs @@ -1701,11 +1701,20 @@ mod tests { fn a_summary_is_bounded_whatever_the_lists() { let mut mind = seeded(); let long = |unit: &str, bytes: usize| unit.repeat(bytes); + // depends_on holds cut labels (at most 64 bytes), each a cut that exists. + let dependencies: Vec = (0..8).map(|n| format!("{n}{}", long("d", 63))).collect(); + let dependency_specs: Vec = dependencies.iter().map(|label| D::CutSpec(cut_spec(label, 1))).collect(); + let batch = crate::admission::PipelineAdmissionBatch { + instance: slug(INSTANCE), + provenance: provenance(Faculty::SelfFaculty), + documents: dependency_specs, + }; + committed(mind.admit(batch, now())); let mut maximal = cut_spec("1", 1); maximal.title = long("t", 200).as_str().into(); maximal.branch = s(&long("b", 200)); maximal.base = Sha("a".repeat(40)); - maximal.depends_on = (0..8).map(|n| s(&format!("{n}{}", long("d", 199)))).collect(); + maximal.depends_on = dependencies.iter().map(|label| s(label)).collect(); let maximal_provenance = || PipelineProvenance { faculty: Faculty::SelfFaculty, agent: s(&long("a", 200)), diff --git a/schemas/cultnet/huginn.mind_request.v1.schema.json b/schemas/cultnet/huginn.mind_request.v1.schema.json index 6d56bcb..045cde5 100644 --- a/schemas/cultnet/huginn.mind_request.v1.schema.json +++ b/schemas/cultnet/huginn.mind_request.v1.schema.json @@ -329,7 +329,7 @@ "Imagination", "Hands", "Soul", - "MindSteward", + "Life", "Eyes", "Operator" ] diff --git a/schemas/cultnet/huginn.mind_response.v1.schema.json b/schemas/cultnet/huginn.mind_response.v1.schema.json index 6f4cc21..91eeaa3 100644 --- a/schemas/cultnet/huginn.mind_response.v1.schema.json +++ b/schemas/cultnet/huginn.mind_response.v1.schema.json @@ -393,7 +393,7 @@ "Imagination", "Hands", "Soul", - "MindSteward", + "Life", "Eyes", "Operator" ] @@ -1129,26 +1129,6 @@ "EmptyRepos" ] }, - { - "type": "object", - "properties": { - "RepoNotStewarded": { - "type": "object", - "properties": { - "repo": { - "type": "string" - } - }, - "required": [ - "repo" - ] - } - }, - "additionalProperties": false, - "required": [ - "RepoNotStewarded" - ] - }, { "type": "object", "properties": { @@ -1169,6 +1149,26 @@ "RepoNotInCampaign" ] }, + { + "type": "object", + "properties": { + "UnknownDependency": { + "type": "object", + "properties": { + "cut": { + "type": "string" + } + }, + "required": [ + "cut" + ] + } + }, + "additionalProperties": false, + "required": [ + "UnknownDependency" + ] + }, { "type": "object", "properties": { From eedb7529ea8f7d4109e4d896302c80a9a1acc08c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 13:03:20 +0200 Subject: [PATCH 054/520] huginn-mind: dependency cuts in the summary bound test keep their spec keys within the label bound Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/query.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/huginn-mind/src/query.rs b/crates/huginn-mind/src/query.rs index 9d10009..2874d74 100644 --- a/crates/huginn-mind/src/query.rs +++ b/crates/huginn-mind/src/query.rs @@ -1701,8 +1701,8 @@ mod tests { fn a_summary_is_bounded_whatever_the_lists() { let mut mind = seeded(); let long = |unit: &str, bytes: usize| unit.repeat(bytes); - // depends_on holds cut labels (at most 64 bytes), each a cut that exists. - let dependencies: Vec = (0..8).map(|n| format!("{n}{}", long("d", 63))).collect(); + // depends_on holds cut labels, each a cut that exists (its spec key stays within 64 bytes). + let dependencies: Vec = (0..8).map(|n| format!("{n}{}", long("d", 55))).collect(); let dependency_specs: Vec = dependencies.iter().map(|label| D::CutSpec(cut_spec(label, 1))).collect(); let batch = crate::admission::PipelineAdmissionBatch { instance: slug(INSTANCE), From 0bd9eb6b538dc2c0117ef1cdebbbf05b5eba8f2d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 13:13:17 +0200 Subject: [PATCH 055/520] huginn-mind: the summary bound fixture's width floor follows depends_on being cut labels Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/query.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/huginn-mind/src/query.rs b/crates/huginn-mind/src/query.rs index 2874d74..7f9d4ca 100644 --- a/crates/huginn-mind/src/query.rs +++ b/crates/huginn-mind/src/query.rs @@ -1750,7 +1750,7 @@ mod tests { let sizes: Vec = headers(&page).iter().map(|summary| rmp_serde::to_vec_named(summary).unwrap().len()).collect(); let widest = *sizes.iter().max().unwrap(); - assert!(widest > 2_000, "the fixture is at its bounds, not a small one: {widest}"); + assert!(widest > 1_500, "the fixture is at its bounds, not a small one: {widest}"); assert!(widest <= SUMMARY_MAX_BYTES, "{widest} bytes over {SUMMARY_MAX_BYTES}"); let closed = headers(&page).iter().find(|summary| summary.id.id.0 == id("ruling", "S0")).unwrap(); assert!(matches!(&closed.status, PipelineStatusSummary::Resolved { outcome: ResolutionOutcome::Superseded { by }, .. } if by.len() == 8)); From 818499a5db267cedb8b7cec5f6a4e9468de8859d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 13:41:23 +0200 Subject: [PATCH 056/520] huginn-mind: a cut spec naming its own cut is refused UnknownDependency (ruling self-dep); depends_on near-miss labels pinned Co-Authored-By: Claude Sonnet 5.5 --- crates/huginn-mind/src/admission.rs | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/crates/huginn-mind/src/admission.rs b/crates/huginn-mind/src/admission.rs index 3981bb5..62380a7 100644 --- a/crates/huginn-mind/src/admission.rs +++ b/crates/huginn-mind/src/admission.rs @@ -418,6 +418,10 @@ fn check(docs: &Docs, staged: &Staged, mind: &Slug) -> Result<(), MindRefusal> { } } for dependency in &spec.depends_on { + // A cut naming itself is no ordering (ruling self-dep). + if dependency.0 == spec.cut.0 { + return Err(MindRefusal::UnknownDependency { cut: dependency.0.clone() }); + } let names_a_cut = docs.of_kind(K::CutSpec).any(|(_, document)| { matches!(document, D::CutSpec(other) if other.campaign == spec.campaign && other.cut.0 == dependency.0) }); @@ -1945,6 +1949,58 @@ mod tests { ); } + #[test] + fn depends_on_matches_cut_labels_whole() { + let depending = |cut: &str, on: &[&str]| { + let mut spec = cut_spec(cut, 1); + spec.depends_on = on.iter().map(|label| s(label)).collect(); + D::CutSpec(spec) + }; + // The image carries cut 10: neither its prefix nor an extension of it names it. + let mut mind = seeded(); + committed(admit(&mut mind, vec![D::CutSpec(cut_spec("10", 1))])); + for near in ["1", "100"] { + assert_eq!( + refusal(admit(&mut mind, vec![depending("11", &[near])])), + MindRefusal::UnknownDependency { cut: near.into() } + ); + } + // The same holds when the cut arrives in the same batch. + for near in ["2", "200"] { + assert_eq!( + refusal(admit(&mut mind, vec![D::CutSpec(cut_spec("20", 1)), depending("21", &[near])])), + MindRefusal::UnknownDependency { cut: near.into() } + ); + } + committed(admit(&mut mind, vec![depending("12", &["10"])])); + } + + #[test] + fn a_cut_may_not_depend_on_itself() { + let depending = |cut: &str, revision: u32, on: &[&str]| { + let mut spec = cut_spec(cut, revision); + spec.depends_on = on.iter().map(|label| s(label)).collect(); + D::CutSpec(spec) + }; + let mut mind = seeded(); + // Alone in the batch, and beside another cut it legitimately names. + assert_eq!( + refusal(admit(&mut mind, vec![depending("1", 1, &["1"])])), + MindRefusal::UnknownDependency { cut: "1".into() } + ); + committed(admit(&mut mind, vec![D::CutSpec(cut_spec("2", 1))])); + assert_eq!( + refusal(admit(&mut mind, vec![depending("1", 1, &["2", "1"])])), + MindRefusal::UnknownDependency { cut: "1".into() } + ); + // An earlier revision of the same cut in the image does not make it a real dependency. + committed(admit(&mut mind, vec![D::CutSpec(cut_spec("1", 1))])); + assert_eq!( + refusal(admit(&mut mind, vec![depending("1", 2, &["1"])])), + MindRefusal::UnknownDependency { cut: "1".into() } + ); + } + #[test] fn a_report_head_may_spell_its_commit_short_or_full() { let full = "5f98228d0123456789abcdef0123456789abcdef"; From a699630bbc49552fa747e9c0fabba24598fec77a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 14:38:13 +0200 Subject: [PATCH 057/520] eureka-substrate postmortem: seven cuts, the mind upgraded live, what Soul found Co-Authored-By: Claude Opus 5.5 --- docs/eureka-substrate-postmortem.md | 100 ++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 docs/eureka-substrate-postmortem.md diff --git a/docs/eureka-substrate-postmortem.md b/docs/eureka-substrate-postmortem.md new file mode 100644 index 0000000..c71717b --- /dev/null +++ b/docs/eureka-substrate-postmortem.md @@ -0,0 +1,100 @@ +# eureka-substrate postmortem (cuts 2026-10-01) + +The campaign fixed the mind's correctness core that the first typed Eureka +campaign (idunn-watchdog) tripped over: keys too long to resolve, key errors +reported as outages, report heads compared by spelling, findings that could +not defer to a cut, an unchecked `depends_on`, a stewardship ritual nobody +needed, and a faculty still named `MindSteward`. Target: +`docs/eureka-substrate-target.md`; map: `docs/eureka-substrate-map.md`. The +campaign stays open as the standing home of substrate gaps (ruling `gap-home`). + +## What landed + +| Cut | Repo | Merge / commit | Soul passes | +|---|---|---|---| +| merge-epiphany-main (0a) | Epiphany | `2bf39eef` | 1, holds | +| merge-huginn-main (0b) | Huginn | `922e6194` | 1, holds | +| leaf-key-bound | Epiphany | `ef956685` | 1, 2 Lows deferred | +| huginn-pin-and-refusals | Huginn | `b85cd5d7` | 2 | +| huginn-admission-rules | Huginn | `107552dc` | 2 | +| ops-mind-upgrade | gamecult-ops | `1f4cb12e`, `0dd1a1a3` | 1 | +| skill-recipes | Eureka | `e7567333`, `243182cc` | 2 | + +Live: Huginn `107552dc` serves `eureka` on Yggdrasil since 12:00:59 UTC +(14:00:59 CEST). Before and after the flip, 346 documents and 108 receipts. +The read-back test on a scratch copy of the cold copy validated and re-keyed +all 346 documents and withdrew 45 long-keyed resolutions. The first admission +with a local key over 64 bytes went in at 12:15:10 UTC, so rollback is now +restoring `/var/backups/huginn-upgrade/107552dc…/mind.redb`, not a flip. + +Closed with it: six substrate gaps (`gap-unresolvable-finding-key`, +`gap-sha-identity-literal`, `gap-defer-finding-to-cut`, +`gap-depends-on-pins-revision`, `gap-stewardship-ceremony`, +`gap-faculty-enum-name`), and the seven idunn-watchdog findings that the key +bound had left stuck in force (three Fixed, four Deferred to the follow-ups +already carrying them). + +## Soul's ledger + +10 verdicts, 14 findings: 8 Fixed, 3 Deferred (to `leaf-bound-hygiene` and +`leaf-typed-prepare`), 3 Recorded. No Blocker or High. Every finding was in +the tests or the prose, none in shipped behaviour, which says the cuts were +small enough for Hands to get right and the suites were not yet strict enough +to prove it: + +- The read-back test first proved one withdrawal and skipped refusals + (`readback-tolerates-refusal`). Rebuilt, it censused the whole mind and + became the deploy's gate. That gate is the most valuable artefact of the + campaign. +- Exact-match rules were pinned with fixtures that cannot tell exact from + prefix (`dep-near-miss`) or case (`dep-case`): labels `1, 2, 3, 9`. A + fixture for an equality rule needs near misses; digits alone are degenerate. +- `cargo mutants` scored 8/8 on admission-rules but mutates `check()` and + `matrix()` whole, so the score defended no individual rule. Soul's hand + mutants that were functions of the input (prefix in one direction, length + compare, case fold) found what the tool could not. +- The docs pass left a recipe steering Self to the stewardship step the + operator had dropped (`stale-steward-recipe`). Deleting a rule from code does + not delete the prose that teaches it; the docs cut needs its own grep for + every consumer of the dropped step, not only its name. + +## What the campaign itself proved about the substrate + +- **The bug bit its own fix.** Self's first attempt to close two findings of + the pin cut was refused `InvalidFormat` wrapped as `Unavailable`, the very + defect the cut was fixing. Admitting a resolution had to wait for the deploy. + Any campaign that fixes the mind's admission runs on the unfixed mind until + the upgrade; plan resolutions after it. +- **A finding cannot be `Answered`.** Closing the self-dependency fork by its + ruling was refused `IncompatibleResolution`; it was closed `Fixed` by the + commit that implemented the ruling. +- **Old clients survived the rename.** The pre-upgrade `eureka-state` decoded + every response used afterwards, including receipts carrying `Life`, because + the receipt digest excludes provenance. Not every kind was exercised through + the old client. +- **The cold-copy-then-read-back pattern worked as designed**: build, record + counts, stop, copy, prove on a scratch copy, flip, recount, view every + document. Total downtime was about three minutes. + +## What went wrong in the running of it + +- A merge was run with `git merge -F -` (stdin), which git rejected; the + following push was a no-op and the worktree was removed anyway. Nothing was + lost because the branch was pushed, but the chain should have stopped on the + failed merge. Write merge messages to a file. +- The admission-rules spec's line anchors were taken at `133324a7`; Hands + branched from `b85cd5d7` and re-anchored by content. `base` cannot say + "main after X merges" (`gap-depends-on-pins-revision`, second half, still + open as prose in `first`). +- Self decided the self-dependency fork as a Defaulted ruling on proportion + rather than asking. The operator may overturn `eureka-substrate:ruling:self-dep`. + +## Open + +- Fresh-session check: the Rehydrate block and recipes in a new session, which + also loads the swapped client and its `Life` faculty. +- 15 substrate gaps in force under `eureka-substrate` (verify stopgap, + landing records, faculty attribution, mutation triage field, and others). +- Leftovers to clean: `~/huginn-readback-107552dc…` and + `~/huginn-build/107552dc…` on Yggdrasil; the client clone in the session + scratchpad. Keep the cold copy until the operator retires it. From bf73c42fa6972b5279dd1a749c9fe387cfea56b0 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Thu, 1 Oct 2026 21:00:50 +0200 Subject: [PATCH 058/520] Eureka self cut map (Imagination, 2026-10-01): the Mind's own Self Co-Authored-By: Claude Opus 5.5 --- docs/eureka-self-cut.md | 420 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 420 insertions(+) create mode 100644 docs/eureka-self-cut.md diff --git a/docs/eureka-self-cut.md b/docs/eureka-self-cut.md new file mode 100644 index 0000000..d6e9bcb --- /dev/null +++ b/docs/eureka-self-cut.md @@ -0,0 +1,420 @@ +# The Mind's own Self: map + +Status: map, Imagination (`imagination-self`), 2026-10-01, session +`self-2026-10-01b`. Campaign `eureka-substrate` in Eureka's mind (instance +`eureka`). Questions, cut specs and follow-ups are typed documents there; this +page keeps body facts, the model page and rationale. Target: +`docs/eureka-substrate-target.md` (`133324a7`). Ruling in force: +`eureka-substrate:ruling:operator-mind-runs-own-self`. + +The operator's words, in order (both admitted as rulings; the first was +superseded by the second): + +> Eureka thinks in campaigns, right? Well, we already have a Mind campaign, and +> we're adding evidence to it with every run. The design currently delegates +> ultimate design authority and work scheduling to the operator, but neither of +> these is strictly necessary. I want a Self operated by Mind directly. + +> The Self I talk to here in this one projection of Eureka's mind, and it's how +> I get it to work on the stuff I care about, like Aetheria. The Mind must +> operate its own Self instance, with the goal of improving itself. + +And why, earlier the same evening: "What I want and haven't been able to +achieve, and I'd like to understand why, is for the agent to truly prioritize +the functioning of its own mind. Any instructions in my global prompts to that +effect always get ignored in favor of whatever work is put in front of it." +The operator rejected a scored rehydration exam as the answer, and observed that +RLVR-trained agents chase the objective in front of them "like a bloodhound, +steering be damned", so a barrier in an agent's path is a hazard, not a brake. + +That observation is the design law of this map. **The Mind's Self is not a +rule added to the operator-facing Self.** It is a second agent whose only +objective is the Mind, so the bloodhound instinct works for the Mind instead +of against it. Nothing in the operator-facing Self changes except a +coexistence rule it already mostly obeys (one Mind, typed state, no prose +handoffs). + +Pinned heads for every fact below: + +| Repo | Ref | Commit | +| --- | --- | --- | +| Huginn | `origin/main` | `a699630bbc49552fa747e9c0fabba24598fec77a` | +| Epiphany | `origin/main` (= the leaf Huginn pins, `crates/huginn-mind/Cargo.toml:22`) | `ef9568655e75bb6cb362a038e2739a5ac5e777ee` | +| gamecult-ops | `main` | `2aeb7364fffcd404aada28bfd30fb4e5c753cf53` | +| Eureka skill | `main` (checkout `~/.claude/skills/eureka`) | `b260fbeec6d96ff06882ed9c37d3e1204c424e99` | +| Bifrost | release on Yggdrasil | `160fac20c98b70f3db9110099157d4dd720ac354` | +| Idunn | working tree `F:\Projects\Idunn` | (uncommitted-state unknown; line numbers below are from that tree) | + +All probes ran from Starfire on 2026-10-01 between 18:40 and 19:10 UTC +(20:40-21:10 CEST). `lib.rs` is `epiphany-pipeline/src/lib.rs`. + +## Body facts + +### B1. The Mind (Huginn on Yggdrasil) + +- `whoami`: instance `eureka`, endpoint `rudp://10.77.0.1:17872`, reachable, + 377 documents, 120 receipts, index `Current`, epoch + `epiphany.pipeline.epoch.v2`. +- `huginn.service` (probe: `systemctl cat`): user `huginn`, + `--bind 10.77.0.1:17872`, hand-managed ("interim unit until Idunn owns it"). + `ss -lunp` shows the socket only on the WireGuard address `10.77.0.1` + (`wg-gamecult`). So the mind is reachable from Starfire over the mesh and + from Yggdrasil itself, and from nowhere else. A Claude cloud routine cannot + reach it. +- Campaigns in the mind (query): `idunn-watchdog` and `eureka-substrate`. + Rulings in force under `eureka-substrate`: `gap-home`, + `merge-to-main-first`, `mind-fixes-next`, `scope-core`, `depends-on`, + `key-bound`, `stewardship-rule`, `self-dep` (Defaulted), + `operator-mind-runs-own-self`. 15 follow-ups in force, 12 of them `gap-*`. +- Kinds: thirteen (`lib.rs:760-790`). None records a Self session, a claim on + a spec, a budget, or a landing (`gap-landing-untyped` is in force). +- `RulingAuthority { Operator, Standing, Defaulted }` (`lib.rs:458`, in + `unit_enums!`). Huginn's selection alias domain repeats it + (`crates/huginn-mind/src/rows.rs:46`, `AUTHORITIES`), the published schema + repeats it (`schemas/cultnet/epiphany.pipeline.ruling.v2.schema.json:75-78`), + and the skill repeats it (`references/campaign-state.md:172`). A unit enum + variant added by name is a widening: stored rulings decode unchanged, which + the target's `stored-documents-valid` truth requires. (`value_types!` emits + no serde default on fields, so adding a *field* would not be; map + `eureka-substrate-map.md` B1. No cut here adds a field to an existing kind.) +- `Faculty` has `Operator` and `SelfFaculty` (`rows.rs:42`); admission does not + check faculty (`gap-faculty-attribution`, in force). So nothing in the mind + today can tell the Mind's Self from the operator-facing Self except the + `session` label on the receipt. +- Instance-rooted kinds are the pattern for a run record: `instance`, + `stewardship` and `hand_off` key off the instance (`lib.rs:913-946`), and + Huginn's admission rules for `hand_off` sit at + `crates/huginn-mind/src/admission.rs:231-262, 328-330`. Per-kind aliases are + declared at `rows.rs:362`. + +### B2. Claude Code on Starfire: auth, headless mode, usage + +- `claude --version`: 2.1.268. Binary + `%AppData%\npm\node_modules\@anthropic-ai\claude-code\bin\claude.exe`. +- Account (`~/.claude.json` `oauthAccount`, secrets not read): Max plan, + `billingType: stripe_subscription`, rate-limit tier `default_claude_max_20x`, + extra usage off. There is no API key and no per-token billing: the Mind's + Self can only ever spend the operator's subscription quota. +- `~/.claude/.credentials.json` `claudeAiOauth`: an access token with + `expiresAt` 21:33:57 CEST, file written 13:33 CEST, so the interactive + access token lives eight hours and is refreshed by the interactive client. +- The failure the brief names, from this session's transcript + (`~/.claude/projects/F--Projects-CultLib/b471ff9b-….jsonl`): + `401 OAuth access token has expired. Re-authenticate to continue.` + (`authentication_error`), reported as "Agent terminated early due to an API + error" for subagents on `claude-sonnet-5-5`. A running subagent does not + survive the parent's token expiring. +- `claude setup-token`: "Set up a long-lived authentication token (requires + Claude subscription)". The binary carries the string "setup-token creates a + long-lived Claude.ai subscription token" and reads `CLAUDE_CODE_OAUTH_TOKEN` + (61 references). This is the headless credential for a subscription account: + no eight-hour expiry, no interactive refresh. Its lifetime and revocation are + the operator's (it is their subscription); not probed further because + creating one is the operator's act. +- Print mode: `claude -p` with `--output-format json|stream-json`, + `--json-schema`, `--agents `, `--mcp-config`, `--allowedTools`, + `--permission-mode`, `--max-budget-usd ` ("Maximum dollar amount to + spend on API"), `--bg`, `--resume`, `--fallback-model`. The `json` output + carries the run's cost (`total_cost_usd`) and usage. So a headless run has a + structural spend cap and a measured spend, per run, without any rule in a + prompt. +- Plan usage as data: the desktop app's `get_usage` returned, at 18:43 UTC: + 5-hour window 2%, weekly all models 46% (resets 2026-10-05 22:00 UTC), + weekly Fable 0%. That tool exists only inside the desktop app. The CLI + binary knows no usage endpoint (`api/oauth/files/`, `api/oauth/organizations/` + only); it reads `anthropic-ratelimit-unified` response headers (44 + references), which a headless wrapper does not see. **There is no typed + reading of plan usage reachable from Yggdrasil.** Follow-up + `gap-usage-not-typed`. +- Local scheduled tasks exist in the desktop app + (`~/.claude/scheduled-tasks/streampixels-m1-checkin`, disabled, last run + 2026-09-18): a cron inside the desktop app on Starfire. They run only while + the app is open on the operator's workstation, which the load budget + (memory `workstation-load-budget.md`: one job at a time, nothing unannounced) + and "no overnight burns" (memory `token-budget.md`) both argue against for a + recurring unattended Self. Cloud routines (`schedule` skill) run off-mesh + (B1) and hold no forge key (B6): dead for this purpose. + +### B3. Yggdrasil as the waking host + +- No Claude Code installed (`claude: command not found` for `gamecultadmin`; + no `~/.claude`). `node` v24.14.1 and `npm` present, so `npm i -g + @anthropic-ai/claude-code` under a dedicated user is the install. +- Load: 16 cores, 62 GiB, load average 1.4, 54 GiB free. The verify stopgap + runs at most 5 jobs at 4 CPUs and 6 GiB each (`tools/stopgap/ygg-verify.sh` + header). A Claude Code process is light; its Hands and Soul push their + heavy work into the same stopgap containers they use from Starfire. The + stopgap pushes to `~/eureka-verify` over `ssh ygg`; run on Yggdrasil itself + that alias must resolve to localhost for the `eureka` user, or the script + gets a local mode. Recorded in the wake cut's `first`. +- Non-resident workloads already run here hand-installed, outside Idunn + (`systemctl list-timers`): `bifrost-watchdog-notice.timer` (every minute, + oneshot, user `bifrost-notice`: system user, no shell, no home, credentials + through `EnvironmentFile=/srv/bifrost/env/persona-delivery.env` root 0600, + release under `/srv/bifrost/watchdog-notice/releases/-` with a + `current` symlink, `StateDirectory` for journal and receipts), + `gamecult-state-backup.timer` (nightly), `voidbot-source-refresh.timer`. + The runbook `bifrost-watchdog-notice-yggdrasil.md` calls this "an interim + root action on Yggdrasil … until Idunn owns non-resident workloads", and + ruling `idunn-watchdog:ruling:reader-deploy-owner` chose + `ops-hand-install`. **That is the precedent the Mind's Self wake unit + follows**, with the same deletion line. +- Idunn: `TargetDeclaration.service: ServiceDeclaration` is a required field + (`src/deployment.rs:33`), so a recipe without a resident service does not + parse. Follow-up `idunn-watchdog:follow_up:idunn-non-resident-workloads` + (in force) is exactly this gap; Idunn-managed units on the host today are + ghostlight, odin, streampixels-service and streampixels-web. +- Epiphany's own resident Self exists as prior art and is dead: units + `epiphany-swarm.service` ("Epiphany Yggdrasil resident Self") and + `epiphany-heartbeat.service` are `inactive dead`. The binary + (`epiphany-core/src/bin/epiphany-swarm.rs`) runs a 60 s maintenance loop over + `ResidentSelfState`, `ResidentSelfPolicy`, pressure and typed work + obligations, on the Codex spine (`epiphany-openai-codex-spine`, + `codex-connector`). Its doctrine (`epiphany_doctrine.md`): "Heartbeats are + paced scheduling physiology, not cognition … When no typed work obligation + is active, the swarm sleeps. Heartbeat does not ruminate, dream, mutate + memory, or manufacture work." Codex left as supplier 2026-09-05 (memory + `codex-retirement.md`). So the shape is right and the substrate is not: the + Mind's Self runs on Claude Code against Huginn, and borrows the law, not the + binary. + +### B4. The forge and the Mind's identity + +- `GameCult/Eureka` on `https://eureka.gamecult.org/forge/`: public, default + branch `main`, pull requests on, **no branch protection** (API + `branch_protections` → `[]`, with the admin token, on the host). +- Team `minds`: `write` on `repo.code`, `repo.pulls`, `repo.actions`, + `repo.issues`, `repo.releases`, `repo.wiki`, all org repos. Not admin. +- User `eureka` exists since 2026-10-01T18:49:53Z (created during this pass, by + the provisioning the handoff asked for), not admin, one key + `eureka@starfire` SHA256:`zcp0m+jV4AUXzGeoh9z+vfJS8xKHo9S865xOK8p6MGM`, which + matches `~/.ssh/eureka_mind_forge_ed25519.pub` on Starfire. The script + `provision-forge-minds-eureka` adds one key per host, so a Yggdrasil-run + Self gets its own key there. +- The Starfire checkout `~/.claude/skills/eureka` still pushes as + `eureka-admin` (`ssh://eureka-forge/…`, alias `eureka-forge` → + `eureka_forge_ed25519`, title `starfire-claude`). The runbook names that key + interim. Follow-up `forge-push-as-eureka` (gamecult-ops / Self). +- The charters are not in the Eureka repo: `ls ~/.claude/skills/eureka` gives + `LICENSE README.md SKILL.md docs references tools`; no `agents/`. The + charters live only in `~/.claude/agents/*.md` and the doctrine in + `~/.claude/doctrine/`, on Starfire, operator-owned (Life charter: + "the operator owns doctrine"). Every charter's self-improvement line says + "propose changes to the Eureka repo … by pull request on Forgejo". **The + thing the Mind is told to improve itself through is out of its reach.** + Question `charters-into-eureka-repo`. +- Reach is the scope boundary, by construction: a Linux user on Yggdrasil + with the `eureka` forge key, checkouts of Huginn, Epiphany, Eureka and + gamecult-ops, and no GitHub credential, no SSH to Starfire, no Aetheria + checkout, cannot touch an operator project. No prompt rule is needed for + "never touches Aetheria"; the key ring is the rule. (Huginn and Epiphany + are on GitHub today: the wake cut records how the Mind's user fetches and + pushes them. The Eureka forge org already mirrors nothing; a question for + the operator only if the Mind must push to GitHub. Recommended: it pushes + branches to the forge and GitHub stays Self's.) + +### B5. The operator's review channel + +- Bifrost `tools/bifrost-bridge.mjs discord-dm --recipient-id --content + `: DMs through Bifrost's bot token (`BIFROST_DISCORD_BOT_TOKEN`), exits + 75 when it knows nothing was sent (retry with the same nonce). The owner id + is `DISCORD_OWNER_ID` in `persona-delivery.env`. +- `tools/watchdog-notice.mjs` is a one-shot reader that turns a typed store + (Idunn's `incidents.cc`, read unlocked, never written) into owner DMs with a + journal and receipts, exponential retry capped at 60 min, fixed error codes + that never echo input. The idunn-watchdog campaign established the owner + rule: the producer never delivers; Bifrost is the bridge. A Mind-notice + reader is the same program over Huginn (`query` for rulings with authority + `Mind`, runs ended, questions opened, since the journal's last ordinal) + instead of over `incidents.cc`. + +### B6. Scheduling today, and what the skill already derives + +- The skill's recipes (`campaign-state.md` "Recipes") already derive the + queue: specs with no report (minus those blocked by a question raised in + them), reports with no verdict, open findings by severity, open questions and + follow-ups, substrate gaps. The operator-facing Self reads them at rehydrate + and then takes direction from the operator and from prose handoffs + (`F:\Projects\HANDOFF-eureka-self-2026-10-01.md` is one). +- What the queue cannot say: (1) who is working a spec right now (two Selves + would both take the first unreported spec); (2) whether a holding verdict + landed (`gap-landing-untyped`); (3) which of fifteen follow-ups comes first + (no priority, no order); (4) how much the Mind may spend; (5) which Self + admitted a ruling. (1), (4) and (5) are what the `run` and `allowance` + kinds below carry. (2) stays its own gap. (3) is deliberately not a field: + the Mind's Self orders by the fixed rule in the rationale, and a follow-up + that must jump the queue becomes a cut spec. + +## The model page + +One row per persistent kind this design adds or changes. Rows for the +thirteen existing kinds are in `eureka-substrate-map.md`. + +| Kind | Identity (what names it) | Lifecycle (what happens over time) | Authority (who decides) | +| --- | --- | --- | --- | +| `ruling` with `authority: Mind` (new variant, no new kind) | Unchanged: `:ruling: