diff --git a/crates/huginn-daemon/src/bin/huginn-sources.rs b/crates/huginn-daemon/src/bin/huginn-sources.rs index 63115f7..e3b03c7 100644 --- a/crates/huginn-daemon/src/bin/huginn-sources.rs +++ b/crates/huginn-daemon/src/bin/huginn-sources.rs @@ -81,9 +81,14 @@ fn main() -> ExitCode { let mut forge = HttpForge::new(&options.forge_url); let mut journal = |report: &huginn_daemon::sources::Report| println!("{report}"); match refresh(&mut embedder, &mut index, &mut forge, &options.state_root, &mut journal) { - Ok(refreshed) if refreshed.failed.is_empty() => ExitCode::SUCCESS, + Ok(refreshed) if refreshed.failed.is_empty() && refreshed.retirement_withheld.is_none() => ExitCode::SUCCESS, Ok(refreshed) => { - eprintln!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", ")); + if !refreshed.failed.is_empty() { + eprintln!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", ")); + } + if let Some(why) = &refreshed.retirement_withheld { + eprintln!("huginn-sources: no repo was retired: {why}"); + } ExitCode::from(1) } Err(error) => { diff --git a/crates/huginn-daemon/src/sources/forge.rs b/crates/huginn-daemon/src/sources/forge.rs index 60cc24f..b9f8914 100644 --- a/crates/huginn-daemon/src/sources/forge.rs +++ b/crates/huginn-daemon/src/sources/forge.rs @@ -6,8 +6,7 @@ use std::time::Duration; use anyhow::{Context, Result, ensure}; use serde_json::Value; -use super::{Forge, SourceRepo}; -use crate::index::{Http, Method}; +use super::{Catalog, Forge, SourceRepo}; /// The organisation whose repositories are the corpus. pub const ORG: &str = "GameCult"; @@ -19,13 +18,38 @@ const CALL_TIMEOUT: Duration = Duration::from_secs(60); /// API, the unauthenticated view of it: what an anonymous reader may see is /// what is indexed. pub struct HttpForge { - http: Http, + agent: ureq::Agent, base: String, } impl HttpForge { pub fn new(base_url: &str) -> Self { - Self { http: Http::new(base_url, CALL_TIMEOUT), base: base_url.trim_end_matches('/').to_owned() } + let agent: ureq::Agent = + ureq::Agent::config_builder().timeout_global(Some(CALL_TIMEOUT)).http_status_as_error(false).build().into(); + Self { agent, base: base_url.trim_end_matches('/').to_owned() } + } + + /// One GET: the status, the count of repos the forge says it holds + /// (`X-Total-Count`, which only a header carries, so this does not go + /// through `index::Http`), and the body. + fn page(&self, path: &str) -> Result<(u16, Option, String)> { + let url = format!("{}{path}", self.base); + let mut response = self.agent.get(&url).call().with_context(|| format!("no answer from {url}"))?; + let status = response.status().as_u16(); + let total = response.headers().get("x-total-count").and_then(|value| value.to_str().ok()).and_then(|value| value.trim().parse().ok()); + let body = response.body_mut().read_to_string().with_context(|| format!("reading the answer from {url}"))?; + Ok((status, total, body)) + } +} + +/// Why the pages listed are not the whole catalog, or `None` when the repos +/// listed (before any are filtered out) are the count the forge states. A forge +/// that states no count cannot vouch for its list. +pub(crate) fn shortfall(listed: usize, stated: Option) -> Option { + match stated { + None => Some("the forge stated no X-Total-Count".to_owned()), + Some(total) if total == listed => None, + Some(total) => Some(format!("the forge counts {total} repos and {listed} were listed")), } } @@ -45,20 +69,23 @@ pub(crate) fn indexable(page: &Value) -> Result> { } impl Forge for HttpForge { - fn catalog(&mut self) -> Result> { + fn catalog(&mut self) -> Result { let mut found = Vec::new(); + let (mut listed_in_all, mut stated) = (0, None); for page in 1.. { let path = format!("/api/v1/orgs/{ORG}/repos?page={page}&limit={PAGE}"); - let (status, answer) = self.http.request(Method::Get, &path, None)?; + let (status, total, answer) = self.page(&path)?; ensure!(status == 200, "the forge answered {status} to {path}"); + stated = stated.or(total); let parsed: Value = serde_json::from_str(&answer).context("the forge's repo page is not JSON")?; let listed = parsed.as_array().map_or(0, Vec::len); + listed_in_all += listed; found.extend(indexable(&parsed)?); if listed < PAGE { break; } } - Ok(found) + Ok(Catalog { repos: found, shortfall: shortfall(listed_in_all, stated) }) } fn clone_url(&self, name: &str) -> String { @@ -91,8 +118,9 @@ mod tests { assert_eq!(names, ["Open"]); } - /// Serves each page in `pages` once, to the request that names it. - fn serve(pages: Vec) -> String { + /// Serves each page in `pages` once, to the request that names it, with + /// `total` as its `X-Total-Count` when given. + fn serve(pages: Vec, total: Option) -> String { use std::io::{Read, Write}; let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let address = listener.local_addr().unwrap(); @@ -103,25 +131,48 @@ mod tests { let read = stream.read(&mut buffer).unwrap(); let request = String::from_utf8_lossy(&buffer[..read]).into_owned(); let body = if request.contains("page=1&") { &pages[0] } else { &pages[1] }; - write!(stream, "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap(); + let count = total.map_or_else(String::new, |total| format!("x-total-count: {total}\r\n")); + write!(stream, "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\n{count}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap(); } }); format!("http://{address}") } + fn page_of(range: std::ops::Range) -> String { + let repos: Vec = range.map(|n| json!({ "name": format!("r{n}"), "default_branch": "main", "private": false, "archived": false, "mirror": false, "empty": false })).collect(); + Value::Array(repos).to_string() + } + /// The catalog reads every page until one is short: a full first page /// does not end it, and a short second page does; clone URLs name the org. #[test] fn the_catalog_reads_pages_until_a_short_one_and_clones_from_the_org() { - let page = |range: std::ops::Range| { - let repos: Vec = range.map(|n| json!({ "name": format!("r{n}"), "default_branch": "main", "private": false, "archived": false, "mirror": false, "empty": false })).collect(); - Value::Array(repos).to_string() - }; - let base = serve(vec![page(0..PAGE), page(PAGE..PAGE + 3)]); + let base = serve(vec![page_of(0..PAGE), page_of(PAGE..PAGE + 3)], Some(PAGE + 3)); let mut forge = HttpForge::new(&format!("{base}/")); - let names: Vec = forge.catalog().unwrap().into_iter().map(|repo| repo.name).collect(); + let catalog = forge.catalog().unwrap(); + let names: Vec = catalog.repos.into_iter().map(|repo| repo.name).collect(); assert_eq!(names.len(), PAGE + 3); assert_eq!((names[0].as_str(), names[PAGE + 2].as_str()), ("r0", "r52")); + assert_eq!(catalog.shortfall, None, "the pages sum to the forge's count"); assert_eq!(forge.clone_url("Demo"), format!("{base}/GameCult/Demo.git")); } + + /// A catalog whose pages do not sum to the count the forge states is + /// marked short: a forge that caps a page below ours stops the paging at + /// the first short page, and an empty answer or a missing count vouches for + /// nothing. + #[test] + fn a_catalog_that_does_not_sum_to_the_forges_count_is_short() { + let truncated = serve(vec![page_of(0..30)], Some(62)); + let catalog = HttpForge::new(&truncated).catalog().unwrap(); + assert_eq!(catalog.repos.len(), 30); + assert!(catalog.shortfall.as_deref().unwrap().contains("counts 62 repos and 30 were listed")); + + let empty = serve(vec!["[]".into()], Some(62)); + let catalog = HttpForge::new(&empty).catalog().unwrap(); + assert!(catalog.repos.is_empty() && catalog.shortfall.is_some()); + + let unstated = serve(vec![page_of(0..3)], None); + assert!(HttpForge::new(&unstated).catalog().unwrap().shortfall.unwrap().contains("X-Total-Count")); + } } diff --git a/crates/huginn-daemon/src/sources/mod.rs b/crates/huginn-daemon/src/sources/mod.rs index e5c7c9f..99ec508 100644 --- a/crates/huginn-daemon/src/sources/mod.rs +++ b/crates/huginn-daemon/src/sources/mod.rs @@ -131,9 +131,19 @@ pub struct SourceRepo { pub default_branch: String, } +/// The repos the forge listed as indexable, and whether the list is the whole +/// of what the forge holds. A list that is not whole cannot say which repos +/// are gone. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Catalog { + pub repos: Vec, + /// Why the list may be missing repos; `None` when it is the forge's count. + pub shortfall: Option, +} + /// The forge: which repos exist, and where to fetch them from. pub trait Forge { - fn catalog(&mut self) -> Result>; + fn catalog(&mut self) -> Result; fn clone_url(&self, name: &str) -> String; } @@ -180,10 +190,13 @@ fn extension(path: &str) -> String { name.rsplit_once('.').map(|(_, extension)| extension.to_ascii_lowercase()).unwrap_or_default() } -/// The names the refresh reports as failed; empty when every repo converged. +/// What a refresh reports: the names that failed, and why nothing was retired +/// if the catalog could not be trusted to say which repos are gone. #[derive(Debug, Default)] pub struct Refreshed { pub failed: Vec, + /// Set when retirement was skipped because the catalog was empty or short. + pub retirement_withheld: Option, } /// Brings the collection to the forge's current state. A collection that @@ -224,10 +237,24 @@ pub fn refresh( let catalog = forge.catalog().context("listing the forge's repos")?; let mirrors = mirrors_dir(state_root); - retire(index, &catalog, &mirrors, &mut cursors)?; - let mut refreshed = Refreshed::default(); - for repo in &catalog { + // Retirement reads the catalog as the complete list of what exists. An + // empty or short list would retire everything it fails to name, so then + // nothing is retired and the run says so. + let withheld = match (&catalog.shortfall, catalog.repos.is_empty()) { + (Some(why), _) => Some(format!("the forge's catalog is incomplete ({why})")), + (None, true) => Some("the forge's catalog lists no indexable repos".to_owned()), + (None, false) => None, + }; + match withheld { + Some(why) => { + eprintln!("huginn-sources: retiring nothing: {why}"); + refreshed.retirement_withheld = Some(why); + } + None => retire(index, &catalog.repos, &mirrors, &mut cursors)?, + } + + for repo in &catalog.repos { let started = Instant::now(); match refresh_repo(embedder, index, forge, &mirrors, repo, &identity.dimensions, &mut cursors) { Ok(mut report) => { diff --git a/crates/huginn-daemon/src/sources/tests.rs b/crates/huginn-daemon/src/sources/tests.rs index 5c87238..f6c6a8d 100644 --- a/crates/huginn-daemon/src/sources/tests.rs +++ b/crates/huginn-daemon/src/sources/tests.rs @@ -58,6 +58,7 @@ impl Remote { struct FakeForge { repos: Vec<(SourceRepo, PathBuf)>, + shortfall: Option, } impl FakeForge { @@ -67,13 +68,14 @@ impl FakeForge { .iter() .map(|(name, remote)| (SourceRepo { name: (*name).into(), default_branch: "main".into() }, remote.path())) .collect(), + shortfall: None, } } } impl Forge for FakeForge { - fn catalog(&mut self) -> Result> { - Ok(self.repos.iter().map(|(repo, _)| repo.clone()).collect()) + fn catalog(&mut self) -> Result { + Ok(Catalog { repos: self.repos.iter().map(|(repo, _)| repo.clone()).collect(), shortfall: self.shortfall.clone() }) } fn clone_url(&self, name: &str) -> String { @@ -304,6 +306,37 @@ fn retired_repo_leaves_nothing() { assert!(!rig.mirror("Gone").exists() && rig.mirror("Keep").exists()); } +/// An empty catalog, and a catalog the forge says is short, retire nothing and +/// the run says why; the same forge answering whole retires what it dropped. +#[test] +fn an_empty_or_short_catalog_retires_nothing() { + let (keep, gone) = (Remote::new(), Remote::new()); + let k = keep.commit(&[("a.rs", b"a\n")], &[]); + gone.commit(&[("b.rs", b"b\n")], &[]); + let mut rig = Rig::new(); + rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("Gone", &gone)])); + let both = rig.index.held(); + + let mut empty = FakeForge::of(&[]); + let (refreshed, reports) = rig.run(&mut empty); + let refreshed = refreshed.unwrap(); + assert!(reports.is_empty() && refreshed.retirement_withheld.as_deref().unwrap().contains("no indexable repos")); + assert_eq!(rig.index.held(), both, "an empty answer deleted nothing"); + assert!(rig.mirror("Gone").exists() && rig.mirror("Keep").exists()); + assert_eq!(rig.index.cursors().len(), 2); + + let mut short = FakeForge::of(&[("Keep", &keep)]); + short.shortfall = Some("the forge counts 2 repos and 1 were listed".into()); + let (refreshed, _) = rig.run(&mut short); + assert!(refreshed.unwrap().retirement_withheld.unwrap().contains("incomplete")); + assert_eq!(rig.index.held(), both, "a short answer deleted nothing"); + assert!(rig.mirror("Gone").exists()); + + let (refreshed, _) = rig.run(&mut FakeForge::of(&[("Keep", &keep)])); + assert_eq!(refreshed.unwrap().retirement_withheld, None); + assert_eq!(rig.index.held(), [held("Keep", "a.rs", 0, &k)], "a whole answer retires what the forge dropped"); +} + /// A mirror no cursor knows (a first ingest that never finished) is retired /// with its points too. #[test]