sources: a retire error skips that repo; a case-only rename keeps no stale points
Retirement aborted the run on the first repo whose delete failed; it now logs, names the repo in Refreshed.failed and goes on, leaving that repo's cursor and mirror to be found again. Names are compared as written, so a repo the forge now spells in another case is retired under its old spelling instead of kept by a case-insensitive match while its points carried the old name (Soul retire-edge-cases). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
6c1cef8729
commit
e47532c122
3 changed files with 69 additions and 10 deletions
|
|
@ -258,6 +258,8 @@ pub(crate) struct SourcesState {
|
|||
pub(crate) fail_upserts_after: Option<u32>,
|
||||
pub(crate) cursor_writes: Vec<std::collections::BTreeMap<String, String>>,
|
||||
pub(crate) deletes: Vec<(String, Option<Vec<String>>)>,
|
||||
/// A delete naming this repo refuses.
|
||||
pub(crate) fail_deletes_of: Option<String>,
|
||||
/// The repo filter and limit of every search, in order.
|
||||
pub(crate) searches: Vec<(Option<String>, u32)>,
|
||||
}
|
||||
|
|
@ -327,6 +329,9 @@ impl SourceIndex for FakeSources {
|
|||
|
||||
fn delete_by(&mut self, repo: &str, paths: Option<&[String]>) -> Result<()> {
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if state.fail_deletes_of.as_deref() == Some(repo) {
|
||||
bail!("the vector store refused the delete");
|
||||
}
|
||||
state.deletes.push((repo.to_owned(), paths.map(<[String]>::to_vec)));
|
||||
state.points.retain(|_, point| {
|
||||
!(point.payload.repo == repo && paths.is_none_or(|paths| paths.contains(&point.payload.path)))
|
||||
|
|
|
|||
|
|
@ -251,7 +251,7 @@ pub fn refresh<E: Embedder, S: SourceIndex, F: Forge>(
|
|||
eprintln!("huginn-sources: retiring nothing: {why}");
|
||||
refreshed.retirement_withheld = Some(why);
|
||||
}
|
||||
None => retire(index, &catalog.repos, &mirrors, &mut cursors)?,
|
||||
None => retire(index, &catalog.repos, &mirrors, &mut cursors, &mut refreshed.failed),
|
||||
}
|
||||
|
||||
for repo in &catalog.repos {
|
||||
|
|
@ -272,13 +272,17 @@ pub fn refresh<E: Embedder, S: SourceIndex, F: Forge>(
|
|||
|
||||
/// Drops every repo the catalog no longer lists: its chunks, its cursor and
|
||||
/// its mirror, in that order, so a failure leaves the cursor to find it again.
|
||||
/// One repo's failure is logged and named in `failed`; the others go on. Names
|
||||
/// are compared as written: a repo renamed only by case is a new repo to the
|
||||
/// index (its points carry the old spelling), so the old one is retired.
|
||||
fn retire<S: SourceIndex>(
|
||||
index: &mut S,
|
||||
catalog: &[SourceRepo],
|
||||
mirrors: &Path,
|
||||
cursors: &mut BTreeMap<String, String>,
|
||||
) -> Result<()> {
|
||||
let listed = |name: &str| catalog.iter().any(|repo| repo.name.eq_ignore_ascii_case(name));
|
||||
failed: &mut Vec<String>,
|
||||
) {
|
||||
let listed = |name: &str| catalog.iter().any(|repo| repo.name == name);
|
||||
let mut gone: BTreeSet<String> = cursors.keys().filter(|name| !listed(name)).cloned().collect();
|
||||
if let Ok(entries) = std::fs::read_dir(mirrors) {
|
||||
for entry in entries.flatten() {
|
||||
|
|
@ -290,15 +294,22 @@ fn retire<S: SourceIndex>(
|
|||
}
|
||||
}
|
||||
for name in gone {
|
||||
index.delete_by(&name, None).with_context(|| format!("deleting the chunks of retired {name}"))?;
|
||||
if cursors.remove(&name).is_some() {
|
||||
index.set_cursors(cursors).context("dropping a retired repo's cursor")?;
|
||||
}
|
||||
let mirror = Mirror::path_for(mirrors, &name);
|
||||
if mirror.exists() {
|
||||
std::fs::remove_dir_all(&mirror).with_context(|| format!("removing {}", mirror.display()))?;
|
||||
if let Err(error) = retire_one(index, &name, mirrors, cursors) {
|
||||
eprintln!("huginn-sources: retiring {name} failed, the others go on: {error:#}");
|
||||
failed.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn retire_one<S: SourceIndex>(index: &mut S, name: &str, mirrors: &Path, cursors: &mut BTreeMap<String, String>) -> Result<()> {
|
||||
index.delete_by(name, None).with_context(|| format!("deleting the chunks of retired {name}"))?;
|
||||
if cursors.remove(name).is_some() {
|
||||
index.set_cursors(cursors).context("dropping a retired repo's cursor")?;
|
||||
}
|
||||
let mirror = Mirror::path_for(mirrors, name);
|
||||
if mirror.exists() {
|
||||
std::fs::remove_dir_all(&mirror).with_context(|| format!("removing {}", mirror.display()))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -337,6 +337,49 @@ fn an_empty_or_short_catalog_retires_nothing() {
|
|||
assert_eq!(rig.index.held(), [held("Keep", "a.rs", 0, &k)], "a whole answer retires what the forge dropped");
|
||||
}
|
||||
|
||||
/// A retirement that fails is that repo's failure alone: it is named, the
|
||||
/// other retirements and the live repos go on, and the failed one keeps its
|
||||
/// cursor and mirror so the next run finds it again.
|
||||
#[test]
|
||||
fn a_retire_error_skips_that_repo_and_the_run_goes_on() {
|
||||
let (keep, first, second) = (Remote::new(), Remote::new(), Remote::new());
|
||||
let k = keep.commit(&[("a.rs", b"a\n")], &[]);
|
||||
first.commit(&[("b.rs", b"b\n")], &[]);
|
||||
second.commit(&[("c.rs", b"c\n")], &[]);
|
||||
let mut rig = Rig::new();
|
||||
rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("First", &first), ("Second", &second)]));
|
||||
let edited = keep.commit(&[("a.rs", b"a edited\n")], &[]);
|
||||
rig.index.state.lock().unwrap().fail_deletes_of = Some("First".into());
|
||||
|
||||
let (refreshed, reports) = rig.run(&mut FakeForge::of(&[("Keep", &keep)]));
|
||||
assert_eq!(refreshed.unwrap().failed, ["First"]);
|
||||
assert!(!rig.mirror("Second").exists() && rig.mirror("First").exists());
|
||||
assert_eq!(rig.index.cursors().into_keys().collect::<Vec<_>>(), ["First", "Keep"]);
|
||||
assert_eq!(reports.len(), 1, "the live repo was refreshed after the failed retirement");
|
||||
assert_ne!(k, edited);
|
||||
|
||||
rig.index.state.lock().unwrap().fail_deletes_of = None;
|
||||
rig.converged(&mut FakeForge::of(&[("Keep", &keep)]));
|
||||
assert!(!rig.mirror("First").exists());
|
||||
assert_eq!(rig.index.cursors().into_keys().collect::<Vec<_>>(), ["Keep"]);
|
||||
assert_eq!(rig.index.held(), [held("Keep", "a.rs", 0, &edited)]);
|
||||
}
|
||||
|
||||
/// A repo the forge now spells with another case is another repo to the
|
||||
/// index: the points under the old spelling do not stay behind.
|
||||
#[test]
|
||||
fn a_repo_renamed_only_by_case_keeps_no_stale_points() {
|
||||
let remote = Remote::new();
|
||||
remote.commit(&[("a.rs", b"a\n")], &[]);
|
||||
let mut rig = Rig::new();
|
||||
rig.converged(&mut FakeForge::of(&[("Demo", &remote)]));
|
||||
let head = remote.commit(&[("b.rs", b"b\n")], &[]);
|
||||
rig.converged(&mut FakeForge::of(&[("demo", &remote)]));
|
||||
let repos: BTreeSet<String> = rig.index.held().into_iter().map(|(repo, _, _, _)| repo).collect();
|
||||
assert_eq!(repos, BTreeSet::from(["demo".to_owned()]));
|
||||
assert_eq!(rig.index.cursors().into_iter().collect::<Vec<_>>(), [("demo".to_owned(), head)]);
|
||||
}
|
||||
|
||||
/// A mirror no cursor knows (a first ingest that never finished) is retired
|
||||
/// with its points too.
|
||||
#[test]
|
||||
|
|
|
|||
Loading…
Reference in a new issue