diff --git a/crates/huginn-daemon/src/index/mod.rs b/crates/huginn-daemon/src/index/mod.rs index e75d7c0..ab864c1 100644 --- a/crates/huginn-daemon/src/index/mod.rs +++ b/crates/huginn-daemon/src/index/mod.rs @@ -18,11 +18,37 @@ //! `Embedder` (`ollama`) and `VectorIndex` (`qdrant`). Those two files are the //! xenos boundary; nothing else in the organ speaks HTTP or JSON. //! -//! Stated limits. The inbox is unbounded: it holds at most the documents -//! admitted while the index is unreachable, each of which is also in the mind. -//! The model identity is read at reconciliation only, so a model re-pulled -//! while the daemon runs takes effect at the next start, and vectors of the -//! wrong length are refused rather than written. +//! Reconciliation is not a startup event. After any failed step the worker +//! reconciles again before it writes anything else: a collection lost while the +//! daemon runs is recreated and refilled, and a model re-pulled with other +//! dimensions rebuilds the collection, all without a restart. Refilling needs +//! the text, so the projector keeps every entry it was given (`known`) beside +//! the ids it still owes (`wanted`): the whole indexable mind's text is held in +//! memory for the process's life. +//! +//! A collection belongs to a mind, not to a name. It is labelled with the +//! instance and with the mind's genesis receipt id (`Mind::genesis_receipt_id`), +//! and a collection labelled for another mind of the same instance name, or +//! for none, is refused and left untouched. A mind with no receipt yet has no +//! identity, so nothing is reconciled until its first admission. +//! +//! Stated limits. +//! +//! - The inbox is unbounded: it holds at most the documents admitted while the +//! index is unreachable, each of which is also in the mind. +//! - Vectors of the wrong length are refused rather than written. +//! - The real input limit is the embedding model's context, not the 16 KiB +//! `INDEX_TEXT_MAX_BYTES` bound. `qwen3-embedding:0.6b` has a 4096 token +//! context, so dense text is cut at about 4k tokens and the tail of a long +//! document is not searchable. Ollama truncates silently by default; its +//! `truncate: false` option refuses over-context input instead, which would +//! fail the whole batch forever on one dense document, so it is not set. +//! There is no chunking. +//! - Points are never deleted. Withdrawn and superseded documents are indexed +//! on purpose and the payload carries no status. Cut 11b's semantic read must +//! therefore join every hit back through the mind: drop ids the mind does not +//! hold, and filter by resolution using RS-3's in-force derivation through +//! selection, never by anything in the payload. pub mod ollama; pub mod qdrant; @@ -63,6 +89,8 @@ pub struct ModelIdentity { pub struct CollectionMeta { pub managed_by: String, pub instance: String, + /// The mind's genesis receipt id: the identity the instance name lacks. + pub mind: String, pub model: String, pub model_digest: String, pub dimensions: u32, @@ -165,28 +193,44 @@ pub struct Projector { embedder: E, index: V, instance: String, + /// `None` until the mind's first admission; nothing is reconciled before. + mind: Option, collection: String, - wanted: BTreeMap, + /// Every entry ever given, by document id: refilling a lost collection + /// needs text the mind's writes have already gone past. + known: BTreeMap, + /// The ids in `known` whose points the collection may lack. + wanted: BTreeSet, dimensions: Option, } impl Projector { - pub fn new(embedder: E, index: V, instance: &Slug) -> Self { + pub fn new(embedder: E, index: V, instance: &Slug, mind: Option) -> Self { Self { embedder, index, instance: instance.0.clone(), + mind, collection: collection_name(instance), - wanted: BTreeMap::new(), + known: BTreeMap::new(), + wanted: BTreeSet::new(), dimensions: None, } } + /// The mind's identity, once it has one. It never changes after that. + fn identify(&mut self, mind: String) { + if self.mind.is_none() { + self.mind = Some(mind); + } + } + /// Entries the collection may lack. Idempotent: an entry already wanted is /// the same document. pub fn want(&mut self, entries: impl IntoIterator) { for entry in entries { - self.wanted.insert(entry.id.id.0.clone(), entry); + self.wanted.insert(entry.id.id.0.clone()); + self.known.insert(entry.id.id.0.clone(), entry); } } @@ -196,7 +240,21 @@ impl Projector { /// One step: reconcile first, then one batch at a time until nothing is /// wanted. + /// + /// Any failure forgets the reconciliation: the next attempt starts from + /// the model and the collection as they are then. pub fn advance(&mut self) -> Result { + let stepped = self.step(); + if stepped.is_err() { + self.dimensions = None; + } + stepped + } + + fn step(&mut self) -> Result { + if self.mind.is_none() { + return Ok(Advance::Idle); + } if self.dimensions.is_none() { self.reconcile()?; return Ok(Advance::Progressed); @@ -211,13 +269,17 @@ impl Projector { /// The only place a point is judged missing. Reads the model identity, /// makes the collection agree with it (creating it, or rebuilding it when /// the model, its digest, its dimensions or the text version differ), - /// refuses a collection that is not this instance's, then drops from - /// `wanted` every entry whose point the collection already holds. + /// refuses a collection that is not this mind's, then makes `wanted` every + /// known entry whose point the collection does not hold. pub fn reconcile(&mut self) -> Result<(), StepError> { + let Some(mind) = self.mind.clone() else { + return Err(StepError::Unavailable(anyhow::anyhow!("the mind has no identity yet"))); + }; let identity = self.embedder.model_identity().context("reading the embedding model's identity")?; let wanted = CollectionMeta { managed_by: MANAGED_BY.into(), instance: self.instance.clone(), + mind: mind.clone(), model: identity.name.clone(), model_digest: identity.digest.clone(), dimensions: identity.dimensions, @@ -231,17 +293,19 @@ impl Projector { self.collection ))); } - Described::Labelled(found) if found.managed_by != MANAGED_BY || found.instance != self.instance => { + Described::Labelled(found) + if found.managed_by != MANAGED_BY || found.instance != self.instance || found.mind != mind => + { return Err(StepError::Refused(format!( - "collection {} is managed by {:?} for instance {:?}; this organ serves {:?}", - self.collection, found.managed_by, found.instance, self.instance + "collection {} is managed by {:?} for instance {:?}, mind {:?}; this organ serves instance {:?}, mind {:?}", + self.collection, found.managed_by, found.instance, found.mind, self.instance, mind ))); } Described::Labelled(found) if found != wanted => self.recreate(&wanted)?, Described::Labelled(_) => {} } let present = self.index.ids(&self.collection).context("listing the collection's points")?; - self.wanted.retain(|document_id, _| !present.contains(&point_id(document_id))); + self.wanted = self.known.keys().filter(|document_id| !present.contains(&point_id(document_id))).cloned().collect(); self.dimensions = Some(identity.dimensions); Ok(()) } @@ -259,7 +323,8 @@ impl Projector { fn flush(&mut self) -> Result<()> { let dimensions = self.dimensions.context("flush before reconcile")? as usize; - let batch: Vec = self.wanted.values().take(BATCH).cloned().collect(); + let batch: Vec = + self.wanted.iter().take(BATCH).filter_map(|document_id| self.known.get(document_id)).cloned().collect(); if batch.is_empty() { return Ok(()); } @@ -296,6 +361,7 @@ impl Projector { /// The health after a step that did not fail. pub fn progress_status(&self) -> IndexStatus { match (self.dimensions, self.pending()) { + (None, 0) if self.mind.is_none() => IndexStatus::Current, (None, pending) => IndexStatus::Reconciling { pending }, (Some(_), 0) => IndexStatus::Current, (Some(_), pending) => IndexStatus::Behind { pending }, @@ -335,6 +401,8 @@ impl Backoff { struct Shared { inbox: Vec, + /// The mind's identity, once a commit has revealed it to a mind that had none. + mind: Option, status: IndexStatus, closed: bool, } @@ -345,14 +413,14 @@ fn lock(shared: &Lock) -> MutexGuard<'_, Shared> { shared.0.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) } -/// With no timeout, blocks until the inbox holds something or the sink is -/// closed. With one, sleeps out the backoff whatever arrives: a failing index +/// With no timeout, blocks until the inbox holds something, the mind reveals +/// its identity, or the sink is closed. With one, sleeps out the backoff whatever arrives: a failing index /// is retried on its schedule, not once per admission. fn wait_for_work(shared: &Lock, timeout: Option) { let mut guard = lock(shared); match timeout { None => { - while guard.inbox.is_empty() && !guard.closed { + while guard.inbox.is_empty() && guard.mind.is_none() && !guard.closed { guard = shared.1.wait(guard).unwrap_or_else(|poisoned| poisoned.into_inner()); } } @@ -396,13 +464,16 @@ fn work(mut projector: Projector, shared: Loc let mut failures = 0_u32; let mut delay = backoff.initial; loop { - let inbox = { + let (mind, inbox) = { let mut guard = lock(&shared); if guard.closed { return; } - std::mem::take(&mut guard.inbox) + (guard.mind.take(), std::mem::take(&mut guard.inbox)) }; + if let Some(mind) = mind { + projector.identify(mind); + } projector.want(inbox); // While a failure is being reported, the report stands until an // attempt succeeds: publishing progress between retries would flicker. @@ -442,25 +513,37 @@ fn work(mut projector: Projector, shared: Loc /// it: a call in flight ends with its own timeout or with the process. pub struct WorkerSink { shared: Lock, + /// Whether the worker has been told the mind's identity. + identified: bool, } impl WorkerSink { - /// Starts the worker. `startup` is every indexable document the mind holds - /// now; the worker's first act is to reconcile the collection against it. - pub fn spawn(embedder: E, index: V, instance: &Slug, startup: Vec, backoff: Backoff) -> Self + /// Starts the worker. `mind` is the mind's genesis receipt id, `None` for + /// a mind not yet written. `startup` is every indexable document the mind + /// holds now; the worker's first act is to reconcile the collection + /// against it. + pub fn spawn( + embedder: E, + index: V, + instance: &Slug, + mind: Option, + startup: Vec, + backoff: Backoff, + ) -> Self where E: Embedder + Send + 'static, V: VectorIndex + Send + 'static, { - let mut projector = Projector::new(embedder, index, instance); + let identified = mind.is_some(); + let mut projector = Projector::new(embedder, index, instance, mind); projector.want(startup); let shared: Lock = Arc::new(( - Mutex::new(Shared { inbox: Vec::new(), status: projector.progress_status(), closed: false }), + Mutex::new(Shared { inbox: Vec::new(), mind: None, status: projector.progress_status(), closed: false }), Condvar::new(), )); let worker = Arc::clone(&shared); std::thread::spawn(move || work(projector, worker, backoff)); - Self { shared } + Self { shared, identified } } /// Blocks until the health satisfies `predicate`, and returns it. For @@ -494,10 +577,17 @@ impl Drop for WorkerSink { impl IndexSink for WorkerSink { fn committed(&mut self, mind: &Mind, writes: &[PipelineRef]) -> Result<()> { let entries = mind.index_entries(Some(writes))?; - if entries.is_empty() { + let revealed = if self.identified { None } else { mind.genesis_receipt_id()? }; + if entries.is_empty() && revealed.is_none() { return Ok(()); } - lock(&self.shared).inbox.extend(entries); + let mut guard = lock(&self.shared); + guard.inbox.extend(entries); + if let Some(revealed) = revealed { + guard.mind = Some(revealed); + self.identified = true; + } + drop(guard); self.shared.1.notify_all(); Ok(()) } diff --git a/crates/huginn-daemon/src/index/qdrant.rs b/crates/huginn-daemon/src/index/qdrant.rs index c119f7e..a794ac1 100644 --- a/crates/huginn-daemon/src/index/qdrant.rs +++ b/crates/huginn-daemon/src/index/qdrant.rs @@ -36,6 +36,7 @@ fn metadata_json(meta: &CollectionMeta) -> Value { json!({ "managed_by": meta.managed_by, "instance": meta.instance, + "mind": meta.mind, "model": meta.model, "model_digest": meta.model_digest, "dimensions": meta.dimensions, @@ -45,8 +46,9 @@ fn metadata_json(meta: &CollectionMeta) -> Value { /// What a collection's `config.metadata` says. A collection whose metadata /// does not name a writer and an instance is unlabelled; one that does but is -/// missing the rest reads as an empty model, which will not match and will be -/// rebuilt if the writer and the instance are ours. +/// missing the rest reads as an empty mind or model, which will not match: an +/// empty mind is refused as another's, an empty model is rebuilt when the +/// writer, the instance and the mind are ours. fn described(metadata: &Value) -> Described { let text = |key: &str| metadata[key].as_str().map(str::to_owned); let (Some(managed_by), Some(instance)) = (text("managed_by"), text("instance")) else { @@ -56,6 +58,7 @@ fn described(metadata: &Value) -> Described { Described::Labelled(CollectionMeta { managed_by, instance, + mind: text("mind").unwrap_or_default(), model: text("model").unwrap_or_default(), model_digest: text("model_digest").unwrap_or_default(), dimensions: number("dimensions"), diff --git a/crates/huginn-daemon/src/index/tests.rs b/crates/huginn-daemon/src/index/tests.rs index e35f695..60ef39b 100644 --- a/crates/huginn-daemon/src/index/tests.rs +++ b/crates/huginn-daemon/src/index/tests.rs @@ -23,6 +23,22 @@ fn collection() -> String { collection_name(&slug(INSTANCE)) } +/// A mind identity for tests that project entries without a mind behind them. +const MIND: &str = "mind-commit-first"; + +/// A mind of the same instance name whose first batch differs, so its genesis +/// receipt id does too. +fn other_mind_of_the_same_name() -> (TempDir, Mind) { + let root = tempfile::tempdir().unwrap(); + let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + let (question, _) = question_and_ruling(); + let mut first = campaign_seed(); + first.push(question); + let outcome = mind.admit(batch(INSTANCE, first), now()); + assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}"); + (root, mind) +} + /// A real mind holding a campaign, a question, the ruling that answers it and /// the resolution admission derived: four indexable documents, beside the /// identity and the stewardship, which carry no text. @@ -52,7 +68,7 @@ fn point_ids(entries: &[IndexEntry]) -> BTreeSet { } fn projected(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector { - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE)); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); projector.want(entries.iter().cloned()); drain(&mut projector); projector @@ -166,9 +182,10 @@ fn a_model_digest_change_rebuilds_the_collection() { fn a_collection_owned_by_another_instance_is_never_recreated() { let (_root, mind) = mind_with_documents(); let entries = mind.index_entries(None).unwrap(); - let ours = |managed_by: &str, instance: &str| CollectionMeta { + let ours = |managed_by: &str, instance: &str, mind: &str| CollectionMeta { managed_by: managed_by.into(), instance: instance.into(), + mind: mind.into(), model: "model".into(), model_digest: "d1".into(), dimensions: 4, @@ -176,8 +193,10 @@ fn a_collection_owned_by_another_instance_is_never_recreated() { }; for foreign in [ Described::Unlabelled, - Described::Labelled(ours(MANAGED_BY, OTHER)), - Described::Labelled(ours("voidbot", INSTANCE)), + Described::Labelled(ours(MANAGED_BY, OTHER, MIND)), + Described::Labelled(ours("voidbot", INSTANCE, MIND)), + Described::Labelled(ours(MANAGED_BY, INSTANCE, "mind-commit-another")), + Described::Labelled(ours(MANAGED_BY, INSTANCE, "")), ] { let (embedder, index) = pair(); index.plant(&collection(), foreign.clone()); @@ -196,7 +215,7 @@ fn a_collection_owned_by_another_instance_is_never_recreated() { }, }, ); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE)); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); projector.want(entries.iter().cloned()); let error = projector.reconcile().expect_err("a foreign collection is refused"); assert!(matches!(error, StepError::Refused(_)), "{foreign:?}: {error}"); @@ -216,7 +235,7 @@ fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() { let (_root, mind) = mind_with_documents(); let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); - let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE)); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into())); projector.want(entries.iter().cloned()); projector.reconcile().unwrap(); embedder.state.lock().unwrap().identity.dimensions = 8; @@ -255,7 +274,7 @@ fn an_admission_replies_while_the_embedder_is_blocked() { let gate = Gate::shut(); let embedder = FakeEmbedder::new("d1").behind(&gate); let index = FakeIndex::default(); - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.index_entries(None).unwrap(), quick()); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), mind.index_entries(None).unwrap(), quick()); let mut daemon = Daemon::new(mind, sink); daemon.index().wait_status(|status| *status == IndexStatus::Current); @@ -283,7 +302,7 @@ fn a_failing_embedder_is_visible_in_whoami_and_retried_never_dropped() { let root = tempfile::tempdir().unwrap(); let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), Vec::new(), quick()); + let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); let mut daemon = Daemon::new(mind, sink); daemon.index().wait_status(|status| *status == IndexStatus::Current); @@ -313,7 +332,7 @@ fn a_refused_collection_is_reported_and_reread() { let entries = mind.index_entries(None).unwrap(); let (embedder, index) = pair(); index.plant(&collection(), Described::Unlabelled); - let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), entries, quick()); + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries, quick()); let mut daemon = Daemon::new(mind, sink); daemon.index().wait_status(|status| matches!(status, IndexStatus::Refused { attempts, .. } if *attempts >= 2)); assert!(matches!(whoami_index(&mut daemon), IndexStatus::Refused { pending: 4, .. })); @@ -452,13 +471,13 @@ fn the_embedders_address_is_not_part_of_the_collections_compatibility() { let repulled = TAGS.replace("ac6d", "beef"); let index = FakeIndex::default(); - let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE)); + let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); first.want(entries.iter().cloned()); drain(&mut first); assert_eq!(counts(&index), (1, 1)); // Another address, the same model and digest. - let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE)); + let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); moved.want(entries.iter().cloned()); drain(&mut moved); assert_eq!(counts(&index), (1, 1)); @@ -466,7 +485,7 @@ fn the_embedders_address_is_not_part_of_the_collections_compatibility() { // The same model name re-pulled: another digest. let leaked: &'static str = Box::leak(repulled.into_boxed_str()); let other = [("GET /api/tags", 200, leaked), routes[1], routes[2]]; - let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE)); + let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into())); rebuilt.want(entries.iter().cloned()); drain(&mut rebuilt); assert_eq!(counts(&index), (2, 2)); @@ -486,13 +505,14 @@ fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { let labelled = stub(&[( "GET /collections/", 200, - r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, + r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","mind":"mind-commit-first","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#, )]); assert_eq!( QdrantIndex::new(&labelled.url).describe(&name).unwrap(), Described::Labelled(CollectionMeta { managed_by: "huginn".into(), instance: "yggdrasil".into(), + mind: "mind-commit-first".into(), model: "m:1".into(), model_digest: "ac6d".into(), dimensions: 4, @@ -517,6 +537,7 @@ fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() { let meta = CollectionMeta { managed_by: MANAGED_BY.into(), instance: INSTANCE.into(), + mind: MIND.into(), model: "m:1".into(), model_digest: "ac6d".into(), dimensions: 4, @@ -590,7 +611,7 @@ fn queued_entries_count_toward_every_status_that_has_a_pending_count() { #[test] fn dropping_the_sink_stops_the_worker() { let (embedder, index) = pair(); - let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Vec::new(), quick()); + let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick()); sink.wait_status(|status| *status == IndexStatus::Current); let shared = Arc::downgrade(&sink.shared); drop(sink); @@ -598,3 +619,133 @@ fn dropping_the_sink_stops_the_worker() { std::thread::yield_now(); } } + +/// F1. The promised curve, pure: 30 s doubling to a 10 min cap, never below +/// the first delay and never past the cap. A retry that shrank would hot-loop +/// against a dead Ollama. +#[test] +fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() { + let backoff = Backoff::default(); + assert_eq!((backoff.initial, backoff.max), (Duration::from_secs(30), Duration::from_secs(600))); + let mut delay = backoff.initial; + let mut seen = vec![delay.as_secs()]; + for _ in 0..7 { + let next = backoff.after(delay); + assert!(next >= delay && next >= backoff.initial, "the delay never shrinks: {delay:?} -> {next:?}"); + assert!(next <= backoff.max, "the delay never passes the cap: {next:?}"); + delay = next; + seen.push(delay.as_secs()); + } + assert_eq!(seen, [30, 60, 120, 240, 480, 600, 600, 600]); + let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3) }; + assert_eq!(small.after(Duration::from_millis(1)), Duration::from_millis(2)); + assert_eq!(small.after(Duration::from_millis(2)), Duration::from_millis(3)); +} + +/// F2. A collection lost while the daemon runs is found by the next failed +/// write, recreated and refilled from what the projector knows, without a +/// restart. +#[test] +fn a_collection_lost_after_startup_is_recreated_and_refilled() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert_eq!(index.holding(&collection()).len(), 3); + + index.state.lock().unwrap().collections.remove(&collection()); + projector.want(entries[3..].iter().cloned()); + let error = projector.advance().expect_err("the write meets no collection"); + assert!(matches!(error, StepError::Unavailable(_)), "{error}"); + assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Failing { pending: 1, .. })); + + drain(&mut projector); + assert_eq!(index.state.lock().unwrap().recreated.len(), 2, "the collection was made again"); + assert_eq!(index.holding(&collection()), point_ids(&entries), "and holds every document, not only the new one"); + assert_eq!(projector.progress_status(), IndexStatus::Current); +} + +/// F2. A model re-pulled with other dimensions fails one write, then the next +/// attempt reads the new identity and rebuilds the collection to match. +#[test] +fn a_model_that_changes_dimensions_after_startup_rebuilds_the_collection() { + let (_root, mind) = mind_with_documents(); + let entries = mind.index_entries(None).unwrap(); + let (embedder, index) = pair(); + let mut projector = projected(&embedder, &index, &entries[..3]); + + embedder.state.lock().unwrap().identity.dimensions = 8; + projector.want(entries[3..].iter().cloned()); + assert!(projector.advance().is_err(), "a vector of 8 does not fit a collection of 4"); + drain(&mut projector); + + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 2); + assert_eq!(state.recreated.last().unwrap().1.dimensions, 8); + let stored = &state.collections[&collection()]; + assert_eq!(stored.points.len(), 4); + assert!(stored.points.values().all(|point| point.vector.len() == 8)); +} + +/// F3. Two minds that share an instance name never share a collection: the +/// second is refused, and the first's points, label and history are untouched. +#[test] +fn a_second_mind_of_the_same_name_is_refused_and_leaves_the_first_alone() { + let (_first_root, first) = mind_with_documents(); + let (_second_root, second) = other_mind_of_the_same_name(); + let (first_id, second_id) = (first.genesis_receipt_id().unwrap().unwrap(), second.genesis_receipt_id().unwrap().unwrap()); + assert_ne!(first_id, second_id); + + let (embedder, index) = pair(); + let mut ours = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(first_id.clone())); + ours.want(first.index_entries(None).unwrap()); + drain(&mut ours); + let held = index.holding(&collection()); + assert_eq!(held.len(), 4); + + let intruder = FakeEmbedder::new("d1"); + let mut theirs = Projector::new(intruder.clone(), index.clone(), &slug(INSTANCE), Some(second_id)); + theirs.want(second.index_entries(None).unwrap()); + let error = theirs.advance().expect_err("another mind's collection is refused"); + assert!(matches!(error, StepError::Refused(_)), "{error}"); + assert!(matches!(theirs.failure_status(&error, 1), IndexStatus::Refused { .. })); + + let state = index.state.lock().unwrap(); + assert_eq!(state.recreated.len(), 1, "never rebuilt by the second mind"); + assert_eq!(state.upserts.len(), 1, "never written by the second mind"); + assert!(intruder.texts().is_empty()); + let Described::Labelled(label) = &state.collections[&collection()].label else { panic!() }; + assert_eq!(label.mind, first_id); + drop(state); + assert_eq!(index.holding(&collection()), held); +} + +/// A mind with no receipt has no identity: nothing is reconciled and no +/// collection is made, until the first admission reveals one. +#[test] +fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() { + let root = tempfile::tempdir().unwrap(); + let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap(); + assert_eq!(mind.genesis_receipt_id().unwrap(), None); + let (embedder, index) = pair(); + let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None); + assert_eq!(projector.progress_status(), IndexStatus::Current); + assert_eq!(projector.advance().unwrap(), Advance::Idle); + assert!(index.state.lock().unwrap().collections.is_empty()); + + let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), None, Vec::new(), quick()); + let mut daemon = Daemon::new(mind, sink); + let seed = campaign_seed(); + let without_text = vec![seed[0].clone(), seed[1].clone()]; + committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, without_text)), now())); + for _ in 0..500 { + if index.state.lock().unwrap().collections.contains_key(&collection()) { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let state = index.state.lock().unwrap(); + let Described::Labelled(label) = &state.collections[&collection()].label else { panic!("no collection was made") }; + assert_eq!(Some(&label.mind), daemon.mind().genesis_receipt_id().unwrap().as_ref()); +} diff --git a/crates/huginn-daemon/src/serve.rs b/crates/huginn-daemon/src/serve.rs index 9a1b91d..287f7b9 100644 --- a/crates/huginn-daemon/src/serve.rs +++ b/crates/huginn-daemon/src/serve.rs @@ -171,7 +171,7 @@ where let mind = Mind::open(&options.state_root, &options.instance)?; let hub = bind(options.bind, &runtime_id(&options.instance))?; let startup_entries = mind.index_entries(None)?; - let sink = WorkerSink::spawn(embedder, index, &options.instance, startup_entries, backoff); + let sink = WorkerSink::spawn(embedder, index, &options.instance, mind.genesis_receipt_id()?, startup_entries, backoff); Ok((Daemon::new(mind, sink), hub, schema_registry()?)) }