sources: retire nothing when the forge's catalog is empty or does not sum to its X-Total-Count
Retirement read the repo list as complete, so an empty or truncated answer would have deleted every repo it failed to name (Soul catalog-shrink-retires). The catalog now carries a shortfall: the pages summed before filtering are checked against X-Total-Count, and a missing header counts as unvouched. The run refuses to retire on a short or empty catalog, refreshes the repos it did see, says so in Refreshed.retirement_withheld, and huginn-sources exits 1. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
3f380df0fe
commit
6c1cef8729
4 changed files with 141 additions and 25 deletions
|
|
@ -81,9 +81,14 @@ fn main() -> ExitCode {
|
|||
let mut forge = HttpForge::new(&options.forge_url);
|
||||
let mut journal = |report: &huginn_daemon::sources::Report| println!("{report}");
|
||||
match refresh(&mut embedder, &mut index, &mut forge, &options.state_root, &mut journal) {
|
||||
Ok(refreshed) if refreshed.failed.is_empty() => ExitCode::SUCCESS,
|
||||
Ok(refreshed) if refreshed.failed.is_empty() && refreshed.retirement_withheld.is_none() => ExitCode::SUCCESS,
|
||||
Ok(refreshed) => {
|
||||
eprintln!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", "));
|
||||
if !refreshed.failed.is_empty() {
|
||||
eprintln!("huginn-sources: {} repos failed: {}", refreshed.failed.len(), refreshed.failed.join(", "));
|
||||
}
|
||||
if let Some(why) = &refreshed.retirement_withheld {
|
||||
eprintln!("huginn-sources: no repo was retired: {why}");
|
||||
}
|
||||
ExitCode::from(1)
|
||||
}
|
||||
Err(error) => {
|
||||
|
|
|
|||
|
|
@ -6,8 +6,7 @@ use std::time::Duration;
|
|||
use anyhow::{Context, Result, ensure};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{Forge, SourceRepo};
|
||||
use crate::index::{Http, Method};
|
||||
use super::{Catalog, Forge, SourceRepo};
|
||||
|
||||
/// The organisation whose repositories are the corpus.
|
||||
pub const ORG: &str = "GameCult";
|
||||
|
|
@ -19,13 +18,38 @@ const CALL_TIMEOUT: Duration = Duration::from_secs(60);
|
|||
/// API, the unauthenticated view of it: what an anonymous reader may see is
|
||||
/// what is indexed.
|
||||
pub struct HttpForge {
|
||||
http: Http,
|
||||
agent: ureq::Agent,
|
||||
base: String,
|
||||
}
|
||||
|
||||
impl HttpForge {
|
||||
pub fn new(base_url: &str) -> Self {
|
||||
Self { http: Http::new(base_url, CALL_TIMEOUT), base: base_url.trim_end_matches('/').to_owned() }
|
||||
let agent: ureq::Agent =
|
||||
ureq::Agent::config_builder().timeout_global(Some(CALL_TIMEOUT)).http_status_as_error(false).build().into();
|
||||
Self { agent, base: base_url.trim_end_matches('/').to_owned() }
|
||||
}
|
||||
|
||||
/// One GET: the status, the count of repos the forge says it holds
|
||||
/// (`X-Total-Count`, which only a header carries, so this does not go
|
||||
/// through `index::Http`), and the body.
|
||||
fn page(&self, path: &str) -> Result<(u16, Option<usize>, String)> {
|
||||
let url = format!("{}{path}", self.base);
|
||||
let mut response = self.agent.get(&url).call().with_context(|| format!("no answer from {url}"))?;
|
||||
let status = response.status().as_u16();
|
||||
let total = response.headers().get("x-total-count").and_then(|value| value.to_str().ok()).and_then(|value| value.trim().parse().ok());
|
||||
let body = response.body_mut().read_to_string().with_context(|| format!("reading the answer from {url}"))?;
|
||||
Ok((status, total, body))
|
||||
}
|
||||
}
|
||||
|
||||
/// Why the pages listed are not the whole catalog, or `None` when the repos
|
||||
/// listed (before any are filtered out) are the count the forge states. A forge
|
||||
/// that states no count cannot vouch for its list.
|
||||
pub(crate) fn shortfall(listed: usize, stated: Option<usize>) -> Option<String> {
|
||||
match stated {
|
||||
None => Some("the forge stated no X-Total-Count".to_owned()),
|
||||
Some(total) if total == listed => None,
|
||||
Some(total) => Some(format!("the forge counts {total} repos and {listed} were listed")),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -45,20 +69,23 @@ pub(crate) fn indexable(page: &Value) -> Result<Vec<SourceRepo>> {
|
|||
}
|
||||
|
||||
impl Forge for HttpForge {
|
||||
fn catalog(&mut self) -> Result<Vec<SourceRepo>> {
|
||||
fn catalog(&mut self) -> Result<Catalog> {
|
||||
let mut found = Vec::new();
|
||||
let (mut listed_in_all, mut stated) = (0, None);
|
||||
for page in 1.. {
|
||||
let path = format!("/api/v1/orgs/{ORG}/repos?page={page}&limit={PAGE}");
|
||||
let (status, answer) = self.http.request(Method::Get, &path, None)?;
|
||||
let (status, total, answer) = self.page(&path)?;
|
||||
ensure!(status == 200, "the forge answered {status} to {path}");
|
||||
stated = stated.or(total);
|
||||
let parsed: Value = serde_json::from_str(&answer).context("the forge's repo page is not JSON")?;
|
||||
let listed = parsed.as_array().map_or(0, Vec::len);
|
||||
listed_in_all += listed;
|
||||
found.extend(indexable(&parsed)?);
|
||||
if listed < PAGE {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(found)
|
||||
Ok(Catalog { repos: found, shortfall: shortfall(listed_in_all, stated) })
|
||||
}
|
||||
|
||||
fn clone_url(&self, name: &str) -> String {
|
||||
|
|
@ -91,8 +118,9 @@ mod tests {
|
|||
assert_eq!(names, ["Open"]);
|
||||
}
|
||||
|
||||
/// Serves each page in `pages` once, to the request that names it.
|
||||
fn serve(pages: Vec<String>) -> String {
|
||||
/// Serves each page in `pages` once, to the request that names it, with
|
||||
/// `total` as its `X-Total-Count` when given.
|
||||
fn serve(pages: Vec<String>, total: Option<usize>) -> String {
|
||||
use std::io::{Read, Write};
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
|
|
@ -103,25 +131,48 @@ mod tests {
|
|||
let read = stream.read(&mut buffer).unwrap();
|
||||
let request = String::from_utf8_lossy(&buffer[..read]).into_owned();
|
||||
let body = if request.contains("page=1&") { &pages[0] } else { &pages[1] };
|
||||
write!(stream, "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap();
|
||||
let count = total.map_or_else(String::new, |total| format!("x-total-count: {total}\r\n"));
|
||||
write!(stream, "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\n{count}content-length: {}\r\nconnection: close\r\n\r\n{body}", body.len()).unwrap();
|
||||
}
|
||||
});
|
||||
format!("http://{address}")
|
||||
}
|
||||
|
||||
fn page_of(range: std::ops::Range<usize>) -> String {
|
||||
let repos: Vec<Value> = range.map(|n| json!({ "name": format!("r{n}"), "default_branch": "main", "private": false, "archived": false, "mirror": false, "empty": false })).collect();
|
||||
Value::Array(repos).to_string()
|
||||
}
|
||||
|
||||
/// The catalog reads every page until one is short: a full first page
|
||||
/// does not end it, and a short second page does; clone URLs name the org.
|
||||
#[test]
|
||||
fn the_catalog_reads_pages_until_a_short_one_and_clones_from_the_org() {
|
||||
let page = |range: std::ops::Range<usize>| {
|
||||
let repos: Vec<Value> = range.map(|n| json!({ "name": format!("r{n}"), "default_branch": "main", "private": false, "archived": false, "mirror": false, "empty": false })).collect();
|
||||
Value::Array(repos).to_string()
|
||||
};
|
||||
let base = serve(vec![page(0..PAGE), page(PAGE..PAGE + 3)]);
|
||||
let base = serve(vec![page_of(0..PAGE), page_of(PAGE..PAGE + 3)], Some(PAGE + 3));
|
||||
let mut forge = HttpForge::new(&format!("{base}/"));
|
||||
let names: Vec<String> = forge.catalog().unwrap().into_iter().map(|repo| repo.name).collect();
|
||||
let catalog = forge.catalog().unwrap();
|
||||
let names: Vec<String> = catalog.repos.into_iter().map(|repo| repo.name).collect();
|
||||
assert_eq!(names.len(), PAGE + 3);
|
||||
assert_eq!((names[0].as_str(), names[PAGE + 2].as_str()), ("r0", "r52"));
|
||||
assert_eq!(catalog.shortfall, None, "the pages sum to the forge's count");
|
||||
assert_eq!(forge.clone_url("Demo"), format!("{base}/GameCult/Demo.git"));
|
||||
}
|
||||
|
||||
/// A catalog whose pages do not sum to the count the forge states is
|
||||
/// marked short: a forge that caps a page below ours stops the paging at
|
||||
/// the first short page, and an empty answer or a missing count vouches for
|
||||
/// nothing.
|
||||
#[test]
|
||||
fn a_catalog_that_does_not_sum_to_the_forges_count_is_short() {
|
||||
let truncated = serve(vec![page_of(0..30)], Some(62));
|
||||
let catalog = HttpForge::new(&truncated).catalog().unwrap();
|
||||
assert_eq!(catalog.repos.len(), 30);
|
||||
assert!(catalog.shortfall.as_deref().unwrap().contains("counts 62 repos and 30 were listed"));
|
||||
|
||||
let empty = serve(vec!["[]".into()], Some(62));
|
||||
let catalog = HttpForge::new(&empty).catalog().unwrap();
|
||||
assert!(catalog.repos.is_empty() && catalog.shortfall.is_some());
|
||||
|
||||
let unstated = serve(vec![page_of(0..3)], None);
|
||||
assert!(HttpForge::new(&unstated).catalog().unwrap().shortfall.unwrap().contains("X-Total-Count"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -131,9 +131,19 @@ pub struct SourceRepo {
|
|||
pub default_branch: String,
|
||||
}
|
||||
|
||||
/// The repos the forge listed as indexable, and whether the list is the whole
|
||||
/// of what the forge holds. A list that is not whole cannot say which repos
|
||||
/// are gone.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Catalog {
|
||||
pub repos: Vec<SourceRepo>,
|
||||
/// Why the list may be missing repos; `None` when it is the forge's count.
|
||||
pub shortfall: Option<String>,
|
||||
}
|
||||
|
||||
/// The forge: which repos exist, and where to fetch them from.
|
||||
pub trait Forge {
|
||||
fn catalog(&mut self) -> Result<Vec<SourceRepo>>;
|
||||
fn catalog(&mut self) -> Result<Catalog>;
|
||||
fn clone_url(&self, name: &str) -> String;
|
||||
}
|
||||
|
||||
|
|
@ -180,10 +190,13 @@ fn extension(path: &str) -> String {
|
|||
name.rsplit_once('.').map(|(_, extension)| extension.to_ascii_lowercase()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The names the refresh reports as failed; empty when every repo converged.
|
||||
/// What a refresh reports: the names that failed, and why nothing was retired
|
||||
/// if the catalog could not be trusted to say which repos are gone.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Refreshed {
|
||||
pub failed: Vec<String>,
|
||||
/// Set when retirement was skipped because the catalog was empty or short.
|
||||
pub retirement_withheld: Option<String>,
|
||||
}
|
||||
|
||||
/// Brings the collection to the forge's current state. A collection that
|
||||
|
|
@ -224,10 +237,24 @@ pub fn refresh<E: Embedder, S: SourceIndex, F: Forge>(
|
|||
|
||||
let catalog = forge.catalog().context("listing the forge's repos")?;
|
||||
let mirrors = mirrors_dir(state_root);
|
||||
retire(index, &catalog, &mirrors, &mut cursors)?;
|
||||
|
||||
let mut refreshed = Refreshed::default();
|
||||
for repo in &catalog {
|
||||
// Retirement reads the catalog as the complete list of what exists. An
|
||||
// empty or short list would retire everything it fails to name, so then
|
||||
// nothing is retired and the run says so.
|
||||
let withheld = match (&catalog.shortfall, catalog.repos.is_empty()) {
|
||||
(Some(why), _) => Some(format!("the forge's catalog is incomplete ({why})")),
|
||||
(None, true) => Some("the forge's catalog lists no indexable repos".to_owned()),
|
||||
(None, false) => None,
|
||||
};
|
||||
match withheld {
|
||||
Some(why) => {
|
||||
eprintln!("huginn-sources: retiring nothing: {why}");
|
||||
refreshed.retirement_withheld = Some(why);
|
||||
}
|
||||
None => retire(index, &catalog.repos, &mirrors, &mut cursors)?,
|
||||
}
|
||||
|
||||
for repo in &catalog.repos {
|
||||
let started = Instant::now();
|
||||
match refresh_repo(embedder, index, forge, &mirrors, repo, &identity.dimensions, &mut cursors) {
|
||||
Ok(mut report) => {
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ impl Remote {
|
|||
|
||||
struct FakeForge {
|
||||
repos: Vec<(SourceRepo, PathBuf)>,
|
||||
shortfall: Option<String>,
|
||||
}
|
||||
|
||||
impl FakeForge {
|
||||
|
|
@ -67,13 +68,14 @@ impl FakeForge {
|
|||
.iter()
|
||||
.map(|(name, remote)| (SourceRepo { name: (*name).into(), default_branch: "main".into() }, remote.path()))
|
||||
.collect(),
|
||||
shortfall: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Forge for FakeForge {
|
||||
fn catalog(&mut self) -> Result<Vec<SourceRepo>> {
|
||||
Ok(self.repos.iter().map(|(repo, _)| repo.clone()).collect())
|
||||
fn catalog(&mut self) -> Result<Catalog> {
|
||||
Ok(Catalog { repos: self.repos.iter().map(|(repo, _)| repo.clone()).collect(), shortfall: self.shortfall.clone() })
|
||||
}
|
||||
|
||||
fn clone_url(&self, name: &str) -> String {
|
||||
|
|
@ -304,6 +306,37 @@ fn retired_repo_leaves_nothing() {
|
|||
assert!(!rig.mirror("Gone").exists() && rig.mirror("Keep").exists());
|
||||
}
|
||||
|
||||
/// An empty catalog, and a catalog the forge says is short, retire nothing and
|
||||
/// the run says why; the same forge answering whole retires what it dropped.
|
||||
#[test]
|
||||
fn an_empty_or_short_catalog_retires_nothing() {
|
||||
let (keep, gone) = (Remote::new(), Remote::new());
|
||||
let k = keep.commit(&[("a.rs", b"a\n")], &[]);
|
||||
gone.commit(&[("b.rs", b"b\n")], &[]);
|
||||
let mut rig = Rig::new();
|
||||
rig.converged(&mut FakeForge::of(&[("Keep", &keep), ("Gone", &gone)]));
|
||||
let both = rig.index.held();
|
||||
|
||||
let mut empty = FakeForge::of(&[]);
|
||||
let (refreshed, reports) = rig.run(&mut empty);
|
||||
let refreshed = refreshed.unwrap();
|
||||
assert!(reports.is_empty() && refreshed.retirement_withheld.as_deref().unwrap().contains("no indexable repos"));
|
||||
assert_eq!(rig.index.held(), both, "an empty answer deleted nothing");
|
||||
assert!(rig.mirror("Gone").exists() && rig.mirror("Keep").exists());
|
||||
assert_eq!(rig.index.cursors().len(), 2);
|
||||
|
||||
let mut short = FakeForge::of(&[("Keep", &keep)]);
|
||||
short.shortfall = Some("the forge counts 2 repos and 1 were listed".into());
|
||||
let (refreshed, _) = rig.run(&mut short);
|
||||
assert!(refreshed.unwrap().retirement_withheld.unwrap().contains("incomplete"));
|
||||
assert_eq!(rig.index.held(), both, "a short answer deleted nothing");
|
||||
assert!(rig.mirror("Gone").exists());
|
||||
|
||||
let (refreshed, _) = rig.run(&mut FakeForge::of(&[("Keep", &keep)]));
|
||||
assert_eq!(refreshed.unwrap().retirement_withheld, None);
|
||||
assert_eq!(rig.index.held(), [held("Keep", "a.rs", 0, &k)], "a whole answer retires what the forge dropped");
|
||||
}
|
||||
|
||||
/// A mirror no cursor knows (a first ingest that never finished) is retired
|
||||
/// with its points too.
|
||||
#[test]
|
||||
|
|
|
|||
Loading…
Reference in a new issue