diff --git a/crates/huginn-daemon/src/index/fakes.rs b/crates/huginn-daemon/src/index/fakes.rs index 0245d46..bf8833f 100644 --- a/crates/huginn-daemon/src/index/fakes.rs +++ b/crates/huginn-daemon/src/index/fakes.rs @@ -258,6 +258,8 @@ pub(crate) struct SourcesState { pub(crate) fail_upserts_after: Option, pub(crate) cursor_writes: Vec>, pub(crate) deletes: Vec<(String, Option>)>, + /// A delete naming this repo refuses. + pub(crate) fail_deletes_of: Option, /// The repo filter and limit of every search, in order. pub(crate) searches: Vec<(Option, u32)>, } @@ -327,6 +329,9 @@ impl SourceIndex for FakeSources { fn delete_by(&mut self, repo: &str, paths: Option<&[String]>) -> Result<()> { let mut state = self.state.lock().unwrap(); + if state.fail_deletes_of.as_deref() == Some(repo) { + bail!("the vector store refused the delete"); + } state.deletes.push((repo.to_owned(), paths.map(<[String]>::to_vec))); state.points.retain(|_, point| { !(point.payload.repo == repo && paths.is_none_or(|paths| paths.contains(&point.payload.path))) diff --git a/crates/huginn-daemon/src/sources/mod.rs b/crates/huginn-daemon/src/sources/mod.rs index 99ec508..b8843ed 100644 --- a/crates/huginn-daemon/src/sources/mod.rs +++ b/crates/huginn-daemon/src/sources/mod.rs @@ -251,7 +251,7 @@ pub fn refresh( eprintln!("huginn-sources: retiring nothing: {why}"); refreshed.retirement_withheld = Some(why); } - None => retire(index, &catalog.repos, &mirrors, &mut cursors)?, + None => retire(index, &catalog.repos, &mirrors, &mut cursors, &mut refreshed.failed), } for repo in &catalog.repos { @@ -272,13 +272,17 @@ pub fn refresh( /// Drops every repo the catalog no longer lists: its chunks, its cursor and /// its mirror, in that order, so a failure leaves the cursor to find it again. +/// One repo's failure is logged and named in `failed`; the others go on. Names +/// are compared as written: a repo renamed only by case is a new repo to the +/// index (its points carry the old spelling), so the old one is retired. fn retire( index: &mut S, catalog: &[SourceRepo], mirrors: &Path, cursors: &mut BTreeMap, -) -> Result<()> { - let listed = |name: &str| catalog.iter().any(|repo| repo.name.eq_ignore_ascii_case(name)); + failed: &mut Vec, +) { + let listed = |name: &str| catalog.iter().any(|repo| repo.name == name); let mut gone: BTreeSet = cursors.keys().filter(|name| !listed(name)).cloned().collect(); if let Ok(entries) = std::fs::read_dir(mirrors) { for entry in entries.flatten() { @@ -290,15 +294,22 @@ fn retire( } } for name in gone { - index.delete_by(&name, None).with_context(|| format!("deleting the chunks of retired {name}"))?; - if cursors.remove(&name).is_some() { - index.set_cursors(cursors).context("dropping a retired repo's cursor")?; - } - let mirror = Mirror::path_for(mirrors, &name); - if mirror.exists() { - std::fs::remove_dir_all(&mirror).with_context(|| format!("removing {}", mirror.display()))?; + if let Err(error) = retire_one(index, &name, mirrors, cursors) { + eprintln!("huginn-sources: retiring {name} failed, the others go on: {error:#}"); + failed.push(name); } } +} + +fn retire_one(index: &mut S, name: &str, mirrors: &Path, cursors: &mut BTreeMap) -> Result<()> { + index.delete_by(name, None).with_context(|| format!("deleting the chunks of retired {name}"))?; + if cursors.remove(name).is_some() { + index.set_cursors(cursors).context("dropping a retired repo's cursor")?; + } + let mirror = Mirror::path_for(mirrors, name); + if mirror.exists() { + std::fs::remove_dir_all(&mirror).with_context(|| format!("removing {}", mirror.display()))?; + } Ok(()) } diff --git a/crates/huginn-daemon/src/sources/tests.rs b/crates/huginn-daemon/src/sources/tests.rs index f6c6a8d..97bce7b 100644 --- a/crates/huginn-daemon/src/sources/tests.rs +++ b/crates/huginn-daemon/src/sources/tests.rs @@ -337,6 +337,49 @@ fn an_empty_or_short_catalog_retires_nothing() { assert_eq!(rig.index.held(), [held("Keep", "a.rs", 0, &k)], "a whole answer retires what the forge dropped"); } +/// A retirement that fails is that repo's failure alone: it is named, the +/// other retirements and the live repos go on, and the failed one keeps its +/// cursor and mirror so the next run finds it again. +#[test] +fn a_retire_error_skips_that_repo_and_the_run_goes_on() { + let (keep, first, second) = (Remote::new(), Remote::new(), Remote::new()); + let k = keep.commit(&[("a.rs", b"a\n")], &[]); + first.commit(&[("b.rs", b"b\n")], &[]); + second.commit(&[("c.rs", b"c\n")], &[]); + let mut rig = Rig::new(); + rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("First", &first), ("Second", &second)])); + let edited = keep.commit(&[("a.rs", b"a edited\n")], &[]); + rig.index.state.lock().unwrap().fail_deletes_of = Some("First".into()); + + let (refreshed, reports) = rig.run(&mut FakeForge::of(&[("Keep", &keep)])); + assert_eq!(refreshed.unwrap().failed, ["First"]); + assert!(!rig.mirror("Second").exists() && rig.mirror("First").exists()); + assert_eq!(rig.index.cursors().into_keys().collect::>(), ["First", "Keep"]); + assert_eq!(reports.len(), 1, "the live repo was refreshed after the failed retirement"); + assert_ne!(k, edited); + + rig.index.state.lock().unwrap().fail_deletes_of = None; + rig.converged(&mut FakeForge::of(&[("Keep", &keep)])); + assert!(!rig.mirror("First").exists()); + assert_eq!(rig.index.cursors().into_keys().collect::>(), ["Keep"]); + assert_eq!(rig.index.held(), [held("Keep", "a.rs", 0, &edited)]); +} + +/// A repo the forge now spells with another case is another repo to the +/// index: the points under the old spelling do not stay behind. +#[test] +fn a_repo_renamed_only_by_case_keeps_no_stale_points() { + let remote = Remote::new(); + remote.commit(&[("a.rs", b"a\n")], &[]); + let mut rig = Rig::new(); + rig.converged(&mut FakeForge::of(&[("Demo", &remote)])); + let head = remote.commit(&[("b.rs", b"b\n")], &[]); + rig.converged(&mut FakeForge::of(&[("demo", &remote)])); + let repos: BTreeSet = rig.index.held().into_iter().map(|(repo, _, _, _)| repo).collect(); + assert_eq!(repos, BTreeSet::from(["demo".to_owned()])); + assert_eq!(rig.index.cursors().into_iter().collect::>(), [("demo".to_owned(), head)]); +} + /// A mirror no cursor knows (a first ingest that never finished) is retired /// with its points too. #[test]