sources: a retire error skips that repo; a case-only rename keeps no stale points

Retirement aborted the run on the first repo whose delete failed; it now logs,
names the repo in Refreshed.failed and goes on, leaving that repo's cursor and
mirror to be found again. Names are compared as written, so a repo the forge
now spells in another case is retired under its old spelling instead of kept by
a case-insensitive match while its points carried the old name (Soul
retire-edge-cases).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Meta Vens 2026-10-31 08:58:19 +01:00
commit e47532c122
3 changed files with 69 additions and 10 deletions

View file

@ -258,6 +258,8 @@ pub(crate) struct SourcesState {
pub(crate) fail_upserts_after: Option<u32>,
pub(crate) cursor_writes: Vec<std::collections::BTreeMap<String, String>>,
pub(crate) deletes: Vec<(String, Option<Vec<String>>)>,
/// A delete naming this repo refuses.
pub(crate) fail_deletes_of: Option<String>,
/// The repo filter and limit of every search, in order.
pub(crate) searches: Vec<(Option<String>, u32)>,
}
@ -327,6 +329,9 @@ impl SourceIndex for FakeSources {
fn delete_by(&mut self, repo: &str, paths: Option<&[String]>) -> Result<()> {
let mut state = self.state.lock().unwrap();
if state.fail_deletes_of.as_deref() == Some(repo) {
bail!("the vector store refused the delete");
}
state.deletes.push((repo.to_owned(), paths.map(<[String]>::to_vec)));
state.points.retain(|_, point| {
!(point.payload.repo == repo && paths.is_none_or(|paths| paths.contains(&point.payload.path)))

View file

@ -251,7 +251,7 @@ pub fn refresh<E: Embedder, S: SourceIndex, F: Forge>(
eprintln!("huginn-sources: retiring nothing: {why}");
refreshed.retirement_withheld = Some(why);
}
None => retire(index, &catalog.repos, &mirrors, &mut cursors)?,
None => retire(index, &catalog.repos, &mirrors, &mut cursors, &mut refreshed.failed),
}
for repo in &catalog.repos {
@ -272,13 +272,17 @@ pub fn refresh<E: Embedder, S: SourceIndex, F: Forge>(
/// Drops every repo the catalog no longer lists: its chunks, its cursor and
/// its mirror, in that order, so a failure leaves the cursor to find it again.
/// One repo's failure is logged and named in `failed`; the others go on. Names
/// are compared as written: a repo renamed only by case is a new repo to the
/// index (its points carry the old spelling), so the old one is retired.
fn retire<S: SourceIndex>(
index: &mut S,
catalog: &[SourceRepo],
mirrors: &Path,
cursors: &mut BTreeMap<String, String>,
) -> Result<()> {
let listed = |name: &str| catalog.iter().any(|repo| repo.name.eq_ignore_ascii_case(name));
failed: &mut Vec<String>,
) {
let listed = |name: &str| catalog.iter().any(|repo| repo.name == name);
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !listed(name)).cloned().collect();
if let Ok(entries) = std::fs::read_dir(mirrors) {
for entry in entries.flatten() {
@ -290,15 +294,22 @@ fn retire<S: SourceIndex>(
}
}
for name in gone {
index.delete_by(&name, None).with_context(|| format!("deleting the chunks of retired {name}"))?;
if cursors.remove(&name).is_some() {
index.set_cursors(cursors).context("dropping a retired repo's cursor")?;
}
let mirror = Mirror::path_for(mirrors, &name);
if mirror.exists() {
std::fs::remove_dir_all(&mirror).with_context(|| format!("removing {}", mirror.display()))?;
if let Err(error) = retire_one(index, &name, mirrors, cursors) {
eprintln!("huginn-sources: retiring {name} failed, the others go on: {error:#}");
failed.push(name);
}
}
}
fn retire_one<S: SourceIndex>(index: &mut S, name: &str, mirrors: &Path, cursors: &mut BTreeMap<String, String>) -> Result<()> {
index.delete_by(name, None).with_context(|| format!("deleting the chunks of retired {name}"))?;
if cursors.remove(name).is_some() {
index.set_cursors(cursors).context("dropping a retired repo's cursor")?;
}
let mirror = Mirror::path_for(mirrors, name);
if mirror.exists() {
std::fs::remove_dir_all(&mirror).with_context(|| format!("removing {}", mirror.display()))?;
}
Ok(())
}

View file

@ -337,6 +337,49 @@ fn an_empty_or_short_catalog_retires_nothing() {
assert_eq!(rig.index.held(), [held("Keep", "a.rs", 0, &k)], "a whole answer retires what the forge dropped");
}
/// A retirement that fails is that repo's failure alone: it is named, the
/// other retirements and the live repos go on, and the failed one keeps its
/// cursor and mirror so the next run finds it again.
#[test]
fn a_retire_error_skips_that_repo_and_the_run_goes_on() {
let (keep, first, second) = (Remote::new(), Remote::new(), Remote::new());
let k = keep.commit(&[("a.rs", b"a\n")], &[]);
first.commit(&[("b.rs", b"b\n")], &[]);
second.commit(&[("c.rs", b"c\n")], &[]);
let mut rig = Rig::new();
rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("First", &first), ("Second", &second)]));
let edited = keep.commit(&[("a.rs", b"a edited\n")], &[]);
rig.index.state.lock().unwrap().fail_deletes_of = Some("First".into());
let (refreshed, reports) = rig.run(&mut FakeForge::of(&[("Keep", &keep)]));
assert_eq!(refreshed.unwrap().failed, ["First"]);
assert!(!rig.mirror("Second").exists() && rig.mirror("First").exists());
assert_eq!(rig.index.cursors().into_keys().collect::<Vec<_>>(), ["First", "Keep"]);
assert_eq!(reports.len(), 1, "the live repo was refreshed after the failed retirement");
assert_ne!(k, edited);
rig.index.state.lock().unwrap().fail_deletes_of = None;
rig.converged(&mut FakeForge::of(&[("Keep", &keep)]));
assert!(!rig.mirror("First").exists());
assert_eq!(rig.index.cursors().into_keys().collect::<Vec<_>>(), ["Keep"]);
assert_eq!(rig.index.held(), [held("Keep", "a.rs", 0, &edited)]);
}
/// A repo the forge now spells with another case is another repo to the
/// index: the points under the old spelling do not stay behind.
#[test]
fn a_repo_renamed_only_by_case_keeps_no_stale_points() {
let remote = Remote::new();
remote.commit(&[("a.rs", b"a\n")], &[]);
let mut rig = Rig::new();
rig.converged(&mut FakeForge::of(&[("Demo", &remote)]));
let head = remote.commit(&[("b.rs", b"b\n")], &[]);
rig.converged(&mut FakeForge::of(&[("demo", &remote)]));
let repos: BTreeSet<String> = rig.index.held().into_iter().map(|(repo, _, _, _)| repo).collect();
assert_eq!(repos, BTreeSet::from(["demo".to_owned()]));
assert_eq!(rig.index.cursors().into_iter().collect::<Vec<_>>(), [("demo".to_owned(), head)]);
}
/// A mirror no cursor knows (a first ingest that never finished) is retired
/// with its points too.
#[test]