Index: reconcile after every failure, label collections with the mind, pin the backoff
F1 the 30 s to 10 min backoff curve is pinned by a pure test. F2 any failed step forgets the reconciliation; the projector keeps every entry it was given so a lost collection or a changed model is rebuilt and refilled without a restart. F3 the collection label carries the mind genesis receipt id; a collection of another mind with the same instance name is refused and untouched. A mind with no receipt yet reconciles nothing. F4 the effective input limit (model context, not 16 KiB) and the Cut 11b join requirement are recorded in the module docs. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
1c1466aa5b
commit
98c959f0f5
4 changed files with 290 additions and 46 deletions
|
|
@ -18,11 +18,37 @@
|
|||
//! `Embedder` (`ollama`) and `VectorIndex` (`qdrant`). Those two files are the
|
||||
//! xenos boundary; nothing else in the organ speaks HTTP or JSON.
|
||||
//!
|
||||
//! Stated limits. The inbox is unbounded: it holds at most the documents
|
||||
//! admitted while the index is unreachable, each of which is also in the mind.
|
||||
//! The model identity is read at reconciliation only, so a model re-pulled
|
||||
//! while the daemon runs takes effect at the next start, and vectors of the
|
||||
//! wrong length are refused rather than written.
|
||||
//! Reconciliation is not a startup event. After any failed step the worker
|
||||
//! reconciles again before it writes anything else: a collection lost while the
|
||||
//! daemon runs is recreated and refilled, and a model re-pulled with other
|
||||
//! dimensions rebuilds the collection, all without a restart. Refilling needs
|
||||
//! the text, so the projector keeps every entry it was given (`known`) beside
|
||||
//! the ids it still owes (`wanted`): the whole indexable mind's text is held in
|
||||
//! memory for the process's life.
|
||||
//!
|
||||
//! A collection belongs to a mind, not to a name. It is labelled with the
|
||||
//! instance and with the mind's genesis receipt id (`Mind::genesis_receipt_id`),
|
||||
//! and a collection labelled for another mind of the same instance name, or
|
||||
//! for none, is refused and left untouched. A mind with no receipt yet has no
|
||||
//! identity, so nothing is reconciled until its first admission.
|
||||
//!
|
||||
//! Stated limits.
|
||||
//!
|
||||
//! - The inbox is unbounded: it holds at most the documents admitted while the
|
||||
//! index is unreachable, each of which is also in the mind.
|
||||
//! - Vectors of the wrong length are refused rather than written.
|
||||
//! - The real input limit is the embedding model's context, not the 16 KiB
|
||||
//! `INDEX_TEXT_MAX_BYTES` bound. `qwen3-embedding:0.6b` has a 4096 token
|
||||
//! context, so dense text is cut at about 4k tokens and the tail of a long
|
||||
//! document is not searchable. Ollama truncates silently by default; its
|
||||
//! `truncate: false` option refuses over-context input instead, which would
|
||||
//! fail the whole batch forever on one dense document, so it is not set.
|
||||
//! There is no chunking.
|
||||
//! - Points are never deleted. Withdrawn and superseded documents are indexed
|
||||
//! on purpose and the payload carries no status. Cut 11b's semantic read must
|
||||
//! therefore join every hit back through the mind: drop ids the mind does not
|
||||
//! hold, and filter by resolution using RS-3's in-force derivation through
|
||||
//! selection, never by anything in the payload.
|
||||
|
||||
pub mod ollama;
|
||||
pub mod qdrant;
|
||||
|
|
@ -63,6 +89,8 @@ pub struct ModelIdentity {
|
|||
pub struct CollectionMeta {
|
||||
pub managed_by: String,
|
||||
pub instance: String,
|
||||
/// The mind's genesis receipt id: the identity the instance name lacks.
|
||||
pub mind: String,
|
||||
pub model: String,
|
||||
pub model_digest: String,
|
||||
pub dimensions: u32,
|
||||
|
|
@ -165,28 +193,44 @@ pub struct Projector<E: Embedder, V: VectorIndex> {
|
|||
embedder: E,
|
||||
index: V,
|
||||
instance: String,
|
||||
/// `None` until the mind's first admission; nothing is reconciled before.
|
||||
mind: Option<String>,
|
||||
collection: String,
|
||||
wanted: BTreeMap<String, IndexEntry>,
|
||||
/// Every entry ever given, by document id: refilling a lost collection
|
||||
/// needs text the mind's writes have already gone past.
|
||||
known: BTreeMap<String, IndexEntry>,
|
||||
/// The ids in `known` whose points the collection may lack.
|
||||
wanted: BTreeSet<String>,
|
||||
dimensions: Option<u32>,
|
||||
}
|
||||
|
||||
impl<E: Embedder, V: VectorIndex> Projector<E, V> {
|
||||
pub fn new(embedder: E, index: V, instance: &Slug) -> Self {
|
||||
pub fn new(embedder: E, index: V, instance: &Slug, mind: Option<String>) -> Self {
|
||||
Self {
|
||||
embedder,
|
||||
index,
|
||||
instance: instance.0.clone(),
|
||||
mind,
|
||||
collection: collection_name(instance),
|
||||
wanted: BTreeMap::new(),
|
||||
known: BTreeMap::new(),
|
||||
wanted: BTreeSet::new(),
|
||||
dimensions: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The mind's identity, once it has one. It never changes after that.
|
||||
fn identify(&mut self, mind: String) {
|
||||
if self.mind.is_none() {
|
||||
self.mind = Some(mind);
|
||||
}
|
||||
}
|
||||
|
||||
/// Entries the collection may lack. Idempotent: an entry already wanted is
|
||||
/// the same document.
|
||||
pub fn want(&mut self, entries: impl IntoIterator<Item = IndexEntry>) {
|
||||
for entry in entries {
|
||||
self.wanted.insert(entry.id.id.0.clone(), entry);
|
||||
self.wanted.insert(entry.id.id.0.clone());
|
||||
self.known.insert(entry.id.id.0.clone(), entry);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -196,7 +240,21 @@ impl<E: Embedder, V: VectorIndex> Projector<E, V> {
|
|||
|
||||
/// One step: reconcile first, then one batch at a time until nothing is
|
||||
/// wanted.
|
||||
///
|
||||
/// Any failure forgets the reconciliation: the next attempt starts from
|
||||
/// the model and the collection as they are then.
|
||||
pub fn advance(&mut self) -> Result<Advance, StepError> {
|
||||
let stepped = self.step();
|
||||
if stepped.is_err() {
|
||||
self.dimensions = None;
|
||||
}
|
||||
stepped
|
||||
}
|
||||
|
||||
fn step(&mut self) -> Result<Advance, StepError> {
|
||||
if self.mind.is_none() {
|
||||
return Ok(Advance::Idle);
|
||||
}
|
||||
if self.dimensions.is_none() {
|
||||
self.reconcile()?;
|
||||
return Ok(Advance::Progressed);
|
||||
|
|
@ -211,13 +269,17 @@ impl<E: Embedder, V: VectorIndex> Projector<E, V> {
|
|||
/// The only place a point is judged missing. Reads the model identity,
|
||||
/// makes the collection agree with it (creating it, or rebuilding it when
|
||||
/// the model, its digest, its dimensions or the text version differ),
|
||||
/// refuses a collection that is not this instance's, then drops from
|
||||
/// `wanted` every entry whose point the collection already holds.
|
||||
/// refuses a collection that is not this mind's, then makes `wanted` every
|
||||
/// known entry whose point the collection does not hold.
|
||||
pub fn reconcile(&mut self) -> Result<(), StepError> {
|
||||
let Some(mind) = self.mind.clone() else {
|
||||
return Err(StepError::Unavailable(anyhow::anyhow!("the mind has no identity yet")));
|
||||
};
|
||||
let identity = self.embedder.model_identity().context("reading the embedding model's identity")?;
|
||||
let wanted = CollectionMeta {
|
||||
managed_by: MANAGED_BY.into(),
|
||||
instance: self.instance.clone(),
|
||||
mind: mind.clone(),
|
||||
model: identity.name.clone(),
|
||||
model_digest: identity.digest.clone(),
|
||||
dimensions: identity.dimensions,
|
||||
|
|
@ -231,17 +293,19 @@ impl<E: Embedder, V: VectorIndex> Projector<E, V> {
|
|||
self.collection
|
||||
)));
|
||||
}
|
||||
Described::Labelled(found) if found.managed_by != MANAGED_BY || found.instance != self.instance => {
|
||||
Described::Labelled(found)
|
||||
if found.managed_by != MANAGED_BY || found.instance != self.instance || found.mind != mind =>
|
||||
{
|
||||
return Err(StepError::Refused(format!(
|
||||
"collection {} is managed by {:?} for instance {:?}; this organ serves {:?}",
|
||||
self.collection, found.managed_by, found.instance, self.instance
|
||||
"collection {} is managed by {:?} for instance {:?}, mind {:?}; this organ serves instance {:?}, mind {:?}",
|
||||
self.collection, found.managed_by, found.instance, found.mind, self.instance, mind
|
||||
)));
|
||||
}
|
||||
Described::Labelled(found) if found != wanted => self.recreate(&wanted)?,
|
||||
Described::Labelled(_) => {}
|
||||
}
|
||||
let present = self.index.ids(&self.collection).context("listing the collection's points")?;
|
||||
self.wanted.retain(|document_id, _| !present.contains(&point_id(document_id)));
|
||||
self.wanted = self.known.keys().filter(|document_id| !present.contains(&point_id(document_id))).cloned().collect();
|
||||
self.dimensions = Some(identity.dimensions);
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -259,7 +323,8 @@ impl<E: Embedder, V: VectorIndex> Projector<E, V> {
|
|||
|
||||
fn flush(&mut self) -> Result<()> {
|
||||
let dimensions = self.dimensions.context("flush before reconcile")? as usize;
|
||||
let batch: Vec<IndexEntry> = self.wanted.values().take(BATCH).cloned().collect();
|
||||
let batch: Vec<IndexEntry> =
|
||||
self.wanted.iter().take(BATCH).filter_map(|document_id| self.known.get(document_id)).cloned().collect();
|
||||
if batch.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
|
@ -296,6 +361,7 @@ impl<E: Embedder, V: VectorIndex> Projector<E, V> {
|
|||
/// The health after a step that did not fail.
|
||||
pub fn progress_status(&self) -> IndexStatus {
|
||||
match (self.dimensions, self.pending()) {
|
||||
(None, 0) if self.mind.is_none() => IndexStatus::Current,
|
||||
(None, pending) => IndexStatus::Reconciling { pending },
|
||||
(Some(_), 0) => IndexStatus::Current,
|
||||
(Some(_), pending) => IndexStatus::Behind { pending },
|
||||
|
|
@ -335,6 +401,8 @@ impl Backoff {
|
|||
|
||||
struct Shared {
|
||||
inbox: Vec<IndexEntry>,
|
||||
/// The mind's identity, once a commit has revealed it to a mind that had none.
|
||||
mind: Option<String>,
|
||||
status: IndexStatus,
|
||||
closed: bool,
|
||||
}
|
||||
|
|
@ -345,14 +413,14 @@ fn lock(shared: &Lock) -> MutexGuard<'_, Shared> {
|
|||
shared.0.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
/// With no timeout, blocks until the inbox holds something or the sink is
|
||||
/// closed. With one, sleeps out the backoff whatever arrives: a failing index
|
||||
/// With no timeout, blocks until the inbox holds something, the mind reveals
|
||||
/// its identity, or the sink is closed. With one, sleeps out the backoff whatever arrives: a failing index
|
||||
/// is retried on its schedule, not once per admission.
|
||||
fn wait_for_work(shared: &Lock, timeout: Option<Duration>) {
|
||||
let mut guard = lock(shared);
|
||||
match timeout {
|
||||
None => {
|
||||
while guard.inbox.is_empty() && !guard.closed {
|
||||
while guard.inbox.is_empty() && guard.mind.is_none() && !guard.closed {
|
||||
guard = shared.1.wait(guard).unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
}
|
||||
}
|
||||
|
|
@ -396,13 +464,16 @@ fn work<E: Embedder, V: VectorIndex>(mut projector: Projector<E, V>, shared: Loc
|
|||
let mut failures = 0_u32;
|
||||
let mut delay = backoff.initial;
|
||||
loop {
|
||||
let inbox = {
|
||||
let (mind, inbox) = {
|
||||
let mut guard = lock(&shared);
|
||||
if guard.closed {
|
||||
return;
|
||||
}
|
||||
std::mem::take(&mut guard.inbox)
|
||||
(guard.mind.take(), std::mem::take(&mut guard.inbox))
|
||||
};
|
||||
if let Some(mind) = mind {
|
||||
projector.identify(mind);
|
||||
}
|
||||
projector.want(inbox);
|
||||
// While a failure is being reported, the report stands until an
|
||||
// attempt succeeds: publishing progress between retries would flicker.
|
||||
|
|
@ -442,25 +513,37 @@ fn work<E: Embedder, V: VectorIndex>(mut projector: Projector<E, V>, shared: Loc
|
|||
/// it: a call in flight ends with its own timeout or with the process.
|
||||
pub struct WorkerSink {
|
||||
shared: Lock,
|
||||
/// Whether the worker has been told the mind's identity.
|
||||
identified: bool,
|
||||
}
|
||||
|
||||
impl WorkerSink {
|
||||
/// Starts the worker. `startup` is every indexable document the mind holds
|
||||
/// now; the worker's first act is to reconcile the collection against it.
|
||||
pub fn spawn<E, V>(embedder: E, index: V, instance: &Slug, startup: Vec<IndexEntry>, backoff: Backoff) -> Self
|
||||
/// Starts the worker. `mind` is the mind's genesis receipt id, `None` for
|
||||
/// a mind not yet written. `startup` is every indexable document the mind
|
||||
/// holds now; the worker's first act is to reconcile the collection
|
||||
/// against it.
|
||||
pub fn spawn<E, V>(
|
||||
embedder: E,
|
||||
index: V,
|
||||
instance: &Slug,
|
||||
mind: Option<String>,
|
||||
startup: Vec<IndexEntry>,
|
||||
backoff: Backoff,
|
||||
) -> Self
|
||||
where
|
||||
E: Embedder + Send + 'static,
|
||||
V: VectorIndex + Send + 'static,
|
||||
{
|
||||
let mut projector = Projector::new(embedder, index, instance);
|
||||
let identified = mind.is_some();
|
||||
let mut projector = Projector::new(embedder, index, instance, mind);
|
||||
projector.want(startup);
|
||||
let shared: Lock = Arc::new((
|
||||
Mutex::new(Shared { inbox: Vec::new(), status: projector.progress_status(), closed: false }),
|
||||
Mutex::new(Shared { inbox: Vec::new(), mind: None, status: projector.progress_status(), closed: false }),
|
||||
Condvar::new(),
|
||||
));
|
||||
let worker = Arc::clone(&shared);
|
||||
std::thread::spawn(move || work(projector, worker, backoff));
|
||||
Self { shared }
|
||||
Self { shared, identified }
|
||||
}
|
||||
|
||||
/// Blocks until the health satisfies `predicate`, and returns it. For
|
||||
|
|
@ -494,10 +577,17 @@ impl Drop for WorkerSink {
|
|||
impl<S: MindStore> IndexSink<S> for WorkerSink {
|
||||
fn committed(&mut self, mind: &Mind<S>, writes: &[PipelineRef]) -> Result<()> {
|
||||
let entries = mind.index_entries(Some(writes))?;
|
||||
if entries.is_empty() {
|
||||
let revealed = if self.identified { None } else { mind.genesis_receipt_id()? };
|
||||
if entries.is_empty() && revealed.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
lock(&self.shared).inbox.extend(entries);
|
||||
let mut guard = lock(&self.shared);
|
||||
guard.inbox.extend(entries);
|
||||
if let Some(revealed) = revealed {
|
||||
guard.mind = Some(revealed);
|
||||
self.identified = true;
|
||||
}
|
||||
drop(guard);
|
||||
self.shared.1.notify_all();
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ fn metadata_json(meta: &CollectionMeta) -> Value {
|
|||
json!({
|
||||
"managed_by": meta.managed_by,
|
||||
"instance": meta.instance,
|
||||
"mind": meta.mind,
|
||||
"model": meta.model,
|
||||
"model_digest": meta.model_digest,
|
||||
"dimensions": meta.dimensions,
|
||||
|
|
@ -45,8 +46,9 @@ fn metadata_json(meta: &CollectionMeta) -> Value {
|
|||
|
||||
/// What a collection's `config.metadata` says. A collection whose metadata
|
||||
/// does not name a writer and an instance is unlabelled; one that does but is
|
||||
/// missing the rest reads as an empty model, which will not match and will be
|
||||
/// rebuilt if the writer and the instance are ours.
|
||||
/// missing the rest reads as an empty mind or model, which will not match: an
|
||||
/// empty mind is refused as another's, an empty model is rebuilt when the
|
||||
/// writer, the instance and the mind are ours.
|
||||
fn described(metadata: &Value) -> Described {
|
||||
let text = |key: &str| metadata[key].as_str().map(str::to_owned);
|
||||
let (Some(managed_by), Some(instance)) = (text("managed_by"), text("instance")) else {
|
||||
|
|
@ -56,6 +58,7 @@ fn described(metadata: &Value) -> Described {
|
|||
Described::Labelled(CollectionMeta {
|
||||
managed_by,
|
||||
instance,
|
||||
mind: text("mind").unwrap_or_default(),
|
||||
model: text("model").unwrap_or_default(),
|
||||
model_digest: text("model_digest").unwrap_or_default(),
|
||||
dimensions: number("dimensions"),
|
||||
|
|
|
|||
|
|
@ -23,6 +23,22 @@ fn collection() -> String {
|
|||
collection_name(&slug(INSTANCE))
|
||||
}
|
||||
|
||||
/// A mind identity for tests that project entries without a mind behind them.
|
||||
const MIND: &str = "mind-commit-first";
|
||||
|
||||
/// A mind of the same instance name whose first batch differs, so its genesis
|
||||
/// receipt id does too.
|
||||
fn other_mind_of_the_same_name() -> (TempDir, Mind<OwnedRedbMessagePackBackingStore>) {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap();
|
||||
let (question, _) = question_and_ruling();
|
||||
let mut first = campaign_seed();
|
||||
first.push(question);
|
||||
let outcome = mind.admit(batch(INSTANCE, first), now());
|
||||
assert!(matches!(outcome, PipelineAdmissionOutcome::Committed { .. }), "{outcome:?}");
|
||||
(root, mind)
|
||||
}
|
||||
|
||||
/// A real mind holding a campaign, a question, the ruling that answers it and
|
||||
/// the resolution admission derived: four indexable documents, beside the
|
||||
/// identity and the stewardship, which carry no text.
|
||||
|
|
@ -52,7 +68,7 @@ fn point_ids(entries: &[IndexEntry]) -> BTreeSet<String> {
|
|||
}
|
||||
|
||||
fn projected(embedder: &FakeEmbedder, index: &FakeIndex, entries: &[IndexEntry]) -> Projector<FakeEmbedder, FakeIndex> {
|
||||
let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE));
|
||||
let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into()));
|
||||
projector.want(entries.iter().cloned());
|
||||
drain(&mut projector);
|
||||
projector
|
||||
|
|
@ -166,9 +182,10 @@ fn a_model_digest_change_rebuilds_the_collection() {
|
|||
fn a_collection_owned_by_another_instance_is_never_recreated() {
|
||||
let (_root, mind) = mind_with_documents();
|
||||
let entries = mind.index_entries(None).unwrap();
|
||||
let ours = |managed_by: &str, instance: &str| CollectionMeta {
|
||||
let ours = |managed_by: &str, instance: &str, mind: &str| CollectionMeta {
|
||||
managed_by: managed_by.into(),
|
||||
instance: instance.into(),
|
||||
mind: mind.into(),
|
||||
model: "model".into(),
|
||||
model_digest: "d1".into(),
|
||||
dimensions: 4,
|
||||
|
|
@ -176,8 +193,10 @@ fn a_collection_owned_by_another_instance_is_never_recreated() {
|
|||
};
|
||||
for foreign in [
|
||||
Described::Unlabelled,
|
||||
Described::Labelled(ours(MANAGED_BY, OTHER)),
|
||||
Described::Labelled(ours("voidbot", INSTANCE)),
|
||||
Described::Labelled(ours(MANAGED_BY, OTHER, MIND)),
|
||||
Described::Labelled(ours("voidbot", INSTANCE, MIND)),
|
||||
Described::Labelled(ours(MANAGED_BY, INSTANCE, "mind-commit-another")),
|
||||
Described::Labelled(ours(MANAGED_BY, INSTANCE, "")),
|
||||
] {
|
||||
let (embedder, index) = pair();
|
||||
index.plant(&collection(), foreign.clone());
|
||||
|
|
@ -196,7 +215,7 @@ fn a_collection_owned_by_another_instance_is_never_recreated() {
|
|||
},
|
||||
},
|
||||
);
|
||||
let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE));
|
||||
let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into()));
|
||||
projector.want(entries.iter().cloned());
|
||||
let error = projector.reconcile().expect_err("a foreign collection is refused");
|
||||
assert!(matches!(error, StepError::Refused(_)), "{foreign:?}: {error}");
|
||||
|
|
@ -216,7 +235,7 @@ fn a_vector_of_the_wrong_length_is_refused_and_nothing_is_dropped() {
|
|||
let (_root, mind) = mind_with_documents();
|
||||
let entries = mind.index_entries(None).unwrap();
|
||||
let (embedder, index) = pair();
|
||||
let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE));
|
||||
let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(MIND.into()));
|
||||
projector.want(entries.iter().cloned());
|
||||
projector.reconcile().unwrap();
|
||||
embedder.state.lock().unwrap().identity.dimensions = 8;
|
||||
|
|
@ -255,7 +274,7 @@ fn an_admission_replies_while_the_embedder_is_blocked() {
|
|||
let gate = Gate::shut();
|
||||
let embedder = FakeEmbedder::new("d1").behind(&gate);
|
||||
let index = FakeIndex::default();
|
||||
let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.index_entries(None).unwrap(), quick());
|
||||
let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), mind.index_entries(None).unwrap(), quick());
|
||||
let mut daemon = Daemon::new(mind, sink);
|
||||
daemon.index().wait_status(|status| *status == IndexStatus::Current);
|
||||
|
||||
|
|
@ -283,7 +302,7 @@ fn a_failing_embedder_is_visible_in_whoami_and_retried_never_dropped() {
|
|||
let root = tempfile::tempdir().unwrap();
|
||||
let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap();
|
||||
let (embedder, index) = pair();
|
||||
let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), Vec::new(), quick());
|
||||
let sink = WorkerSink::spawn(embedder.clone(), index.clone(), &slug(INSTANCE), None, Vec::new(), quick());
|
||||
let mut daemon = Daemon::new(mind, sink);
|
||||
daemon.index().wait_status(|status| *status == IndexStatus::Current);
|
||||
|
||||
|
|
@ -313,7 +332,7 @@ fn a_refused_collection_is_reported_and_reread() {
|
|||
let entries = mind.index_entries(None).unwrap();
|
||||
let (embedder, index) = pair();
|
||||
index.plant(&collection(), Described::Unlabelled);
|
||||
let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), entries, quick());
|
||||
let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), mind.genesis_receipt_id().unwrap(), entries, quick());
|
||||
let mut daemon = Daemon::new(mind, sink);
|
||||
daemon.index().wait_status(|status| matches!(status, IndexStatus::Refused { attempts, .. } if *attempts >= 2));
|
||||
assert!(matches!(whoami_index(&mut daemon), IndexStatus::Refused { pending: 4, .. }));
|
||||
|
|
@ -452,13 +471,13 @@ fn the_embedders_address_is_not_part_of_the_collections_compatibility() {
|
|||
let repulled = TAGS.replace("ac6d", "beef");
|
||||
let index = FakeIndex::default();
|
||||
|
||||
let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE));
|
||||
let mut first = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into()));
|
||||
first.want(entries.iter().cloned());
|
||||
drain(&mut first);
|
||||
assert_eq!(counts(&index), (1, 1));
|
||||
|
||||
// Another address, the same model and digest.
|
||||
let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE));
|
||||
let mut moved = Projector::new(OllamaEmbedder::new(&stub(&routes).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into()));
|
||||
moved.want(entries.iter().cloned());
|
||||
drain(&mut moved);
|
||||
assert_eq!(counts(&index), (1, 1));
|
||||
|
|
@ -466,7 +485,7 @@ fn the_embedders_address_is_not_part_of_the_collections_compatibility() {
|
|||
// The same model name re-pulled: another digest.
|
||||
let leaked: &'static str = Box::leak(repulled.into_boxed_str());
|
||||
let other = [("GET /api/tags", 200, leaked), routes[1], routes[2]];
|
||||
let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE));
|
||||
let mut rebuilt = Projector::new(OllamaEmbedder::new(&stub(&other).url, "m:1"), index.clone(), &slug(INSTANCE), Some(MIND.into()));
|
||||
rebuilt.want(entries.iter().cloned());
|
||||
drain(&mut rebuilt);
|
||||
assert_eq!(counts(&index), (2, 2));
|
||||
|
|
@ -486,13 +505,14 @@ fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() {
|
|||
let labelled = stub(&[(
|
||||
"GET /collections/",
|
||||
200,
|
||||
r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#,
|
||||
r#"{"result":{"config":{"metadata":{"managed_by":"huginn","instance":"yggdrasil","mind":"mind-commit-first","model":"m:1","model_digest":"ac6d","dimensions":4,"index_text_version":1}}}}"#,
|
||||
)]);
|
||||
assert_eq!(
|
||||
QdrantIndex::new(&labelled.url).describe(&name).unwrap(),
|
||||
Described::Labelled(CollectionMeta {
|
||||
managed_by: "huginn".into(),
|
||||
instance: "yggdrasil".into(),
|
||||
mind: "mind-commit-first".into(),
|
||||
model: "m:1".into(),
|
||||
model_digest: "ac6d".into(),
|
||||
dimensions: 4,
|
||||
|
|
@ -517,6 +537,7 @@ fn qdrant_describes_creates_lists_and_writes_what_the_projector_asks() {
|
|||
let meta = CollectionMeta {
|
||||
managed_by: MANAGED_BY.into(),
|
||||
instance: INSTANCE.into(),
|
||||
mind: MIND.into(),
|
||||
model: "m:1".into(),
|
||||
model_digest: "ac6d".into(),
|
||||
dimensions: 4,
|
||||
|
|
@ -590,7 +611,7 @@ fn queued_entries_count_toward_every_status_that_has_a_pending_count() {
|
|||
#[test]
|
||||
fn dropping_the_sink_stops_the_worker() {
|
||||
let (embedder, index) = pair();
|
||||
let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Vec::new(), quick());
|
||||
let sink = WorkerSink::spawn(embedder, index, &slug(INSTANCE), Some(MIND.into()), Vec::new(), quick());
|
||||
sink.wait_status(|status| *status == IndexStatus::Current);
|
||||
let shared = Arc::downgrade(&sink.shared);
|
||||
drop(sink);
|
||||
|
|
@ -598,3 +619,133 @@ fn dropping_the_sink_stops_the_worker() {
|
|||
std::thread::yield_now();
|
||||
}
|
||||
}
|
||||
|
||||
/// F1. The promised curve, pure: 30 s doubling to a 10 min cap, never below
|
||||
/// the first delay and never past the cap. A retry that shrank would hot-loop
|
||||
/// against a dead Ollama.
|
||||
#[test]
|
||||
fn the_backoff_doubles_from_thirty_seconds_to_a_ten_minute_cap() {
|
||||
let backoff = Backoff::default();
|
||||
assert_eq!((backoff.initial, backoff.max), (Duration::from_secs(30), Duration::from_secs(600)));
|
||||
let mut delay = backoff.initial;
|
||||
let mut seen = vec![delay.as_secs()];
|
||||
for _ in 0..7 {
|
||||
let next = backoff.after(delay);
|
||||
assert!(next >= delay && next >= backoff.initial, "the delay never shrinks: {delay:?} -> {next:?}");
|
||||
assert!(next <= backoff.max, "the delay never passes the cap: {next:?}");
|
||||
delay = next;
|
||||
seen.push(delay.as_secs());
|
||||
}
|
||||
assert_eq!(seen, [30, 60, 120, 240, 480, 600, 600, 600]);
|
||||
let small = Backoff { initial: Duration::from_millis(1), max: Duration::from_millis(3) };
|
||||
assert_eq!(small.after(Duration::from_millis(1)), Duration::from_millis(2));
|
||||
assert_eq!(small.after(Duration::from_millis(2)), Duration::from_millis(3));
|
||||
}
|
||||
|
||||
/// F2. A collection lost while the daemon runs is found by the next failed
|
||||
/// write, recreated and refilled from what the projector knows, without a
|
||||
/// restart.
|
||||
#[test]
|
||||
fn a_collection_lost_after_startup_is_recreated_and_refilled() {
|
||||
let (_root, mind) = mind_with_documents();
|
||||
let entries = mind.index_entries(None).unwrap();
|
||||
let (embedder, index) = pair();
|
||||
let mut projector = projected(&embedder, &index, &entries[..3]);
|
||||
assert_eq!(projector.progress_status(), IndexStatus::Current);
|
||||
assert_eq!(index.holding(&collection()).len(), 3);
|
||||
|
||||
index.state.lock().unwrap().collections.remove(&collection());
|
||||
projector.want(entries[3..].iter().cloned());
|
||||
let error = projector.advance().expect_err("the write meets no collection");
|
||||
assert!(matches!(error, StepError::Unavailable(_)), "{error}");
|
||||
assert!(matches!(projector.failure_status(&error, 1), IndexStatus::Failing { pending: 1, .. }));
|
||||
|
||||
drain(&mut projector);
|
||||
assert_eq!(index.state.lock().unwrap().recreated.len(), 2, "the collection was made again");
|
||||
assert_eq!(index.holding(&collection()), point_ids(&entries), "and holds every document, not only the new one");
|
||||
assert_eq!(projector.progress_status(), IndexStatus::Current);
|
||||
}
|
||||
|
||||
/// F2. A model re-pulled with other dimensions fails one write, then the next
|
||||
/// attempt reads the new identity and rebuilds the collection to match.
|
||||
#[test]
|
||||
fn a_model_that_changes_dimensions_after_startup_rebuilds_the_collection() {
|
||||
let (_root, mind) = mind_with_documents();
|
||||
let entries = mind.index_entries(None).unwrap();
|
||||
let (embedder, index) = pair();
|
||||
let mut projector = projected(&embedder, &index, &entries[..3]);
|
||||
|
||||
embedder.state.lock().unwrap().identity.dimensions = 8;
|
||||
projector.want(entries[3..].iter().cloned());
|
||||
assert!(projector.advance().is_err(), "a vector of 8 does not fit a collection of 4");
|
||||
drain(&mut projector);
|
||||
|
||||
let state = index.state.lock().unwrap();
|
||||
assert_eq!(state.recreated.len(), 2);
|
||||
assert_eq!(state.recreated.last().unwrap().1.dimensions, 8);
|
||||
let stored = &state.collections[&collection()];
|
||||
assert_eq!(stored.points.len(), 4);
|
||||
assert!(stored.points.values().all(|point| point.vector.len() == 8));
|
||||
}
|
||||
|
||||
/// F3. Two minds that share an instance name never share a collection: the
|
||||
/// second is refused, and the first's points, label and history are untouched.
|
||||
#[test]
|
||||
fn a_second_mind_of_the_same_name_is_refused_and_leaves_the_first_alone() {
|
||||
let (_first_root, first) = mind_with_documents();
|
||||
let (_second_root, second) = other_mind_of_the_same_name();
|
||||
let (first_id, second_id) = (first.genesis_receipt_id().unwrap().unwrap(), second.genesis_receipt_id().unwrap().unwrap());
|
||||
assert_ne!(first_id, second_id);
|
||||
|
||||
let (embedder, index) = pair();
|
||||
let mut ours = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), Some(first_id.clone()));
|
||||
ours.want(first.index_entries(None).unwrap());
|
||||
drain(&mut ours);
|
||||
let held = index.holding(&collection());
|
||||
assert_eq!(held.len(), 4);
|
||||
|
||||
let intruder = FakeEmbedder::new("d1");
|
||||
let mut theirs = Projector::new(intruder.clone(), index.clone(), &slug(INSTANCE), Some(second_id));
|
||||
theirs.want(second.index_entries(None).unwrap());
|
||||
let error = theirs.advance().expect_err("another mind's collection is refused");
|
||||
assert!(matches!(error, StepError::Refused(_)), "{error}");
|
||||
assert!(matches!(theirs.failure_status(&error, 1), IndexStatus::Refused { .. }));
|
||||
|
||||
let state = index.state.lock().unwrap();
|
||||
assert_eq!(state.recreated.len(), 1, "never rebuilt by the second mind");
|
||||
assert_eq!(state.upserts.len(), 1, "never written by the second mind");
|
||||
assert!(intruder.texts().is_empty());
|
||||
let Described::Labelled(label) = &state.collections[&collection()].label else { panic!() };
|
||||
assert_eq!(label.mind, first_id);
|
||||
drop(state);
|
||||
assert_eq!(index.holding(&collection()), held);
|
||||
}
|
||||
|
||||
/// A mind with no receipt has no identity: nothing is reconciled and no
|
||||
/// collection is made, until the first admission reveals one.
|
||||
#[test]
|
||||
fn a_mind_without_an_identity_makes_no_collection_until_its_first_admission() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mind = Mind::open(root.path(), &slug(INSTANCE)).unwrap();
|
||||
assert_eq!(mind.genesis_receipt_id().unwrap(), None);
|
||||
let (embedder, index) = pair();
|
||||
let mut projector = Projector::new(embedder.clone(), index.clone(), &slug(INSTANCE), None);
|
||||
assert_eq!(projector.progress_status(), IndexStatus::Current);
|
||||
assert_eq!(projector.advance().unwrap(), Advance::Idle);
|
||||
assert!(index.state.lock().unwrap().collections.is_empty());
|
||||
|
||||
let sink = WorkerSink::spawn(embedder, index.clone(), &slug(INSTANCE), None, Vec::new(), quick());
|
||||
let mut daemon = Daemon::new(mind, sink);
|
||||
let seed = campaign_seed();
|
||||
let without_text = vec![seed[0].clone(), seed[1].clone()];
|
||||
committed(daemon.handle(HuginnMindRequest::Admit(batch(INSTANCE, without_text)), now()));
|
||||
for _ in 0..500 {
|
||||
if index.state.lock().unwrap().collections.contains_key(&collection()) {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
let state = index.state.lock().unwrap();
|
||||
let Described::Labelled(label) = &state.collections[&collection()].label else { panic!("no collection was made") };
|
||||
assert_eq!(Some(&label.mind), daemon.mind().genesis_receipt_id().unwrap().as_ref());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ where
|
|||
let mind = Mind::open(&options.state_root, &options.instance)?;
|
||||
let hub = bind(options.bind, &runtime_id(&options.instance))?;
|
||||
let startup_entries = mind.index_entries(None)?;
|
||||
let sink = WorkerSink::spawn(embedder, index, &options.instance, startup_entries, backoff);
|
||||
let sink = WorkerSink::spawn(embedder, index, &options.instance, mind.genesis_receipt_id()?, startup_entries, backoff);
|
||||
Ok((Daemon::new(mind, sink), hub, schema_registry()?))
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue