diff --git a/crates/eureka-state/src/bin/huginn/import.rs b/crates/eureka-state/src/bin/huginn/import.rs index 065688d..ae334c8 100644 --- a/crates/eureka-state/src/bin/huginn/import.rs +++ b/crates/eureka-state/src/bin/huginn/import.rs @@ -12,10 +12,10 @@ use std::collections::{BTreeMap, BTreeSet}; use cultnet_rs::{LIMIT_MAX, PROJECTION_DOCUMENT, Selection}; use eureka_state::hold; -use huginn_mind::eureka_pipeline::{PipelineDocument, PipelineRef, Short, Slug}; +use huginn_mind::eureka_pipeline::{PipelineDocument, PipelineKind, PipelineRef, Short, Slug}; use huginn_mind::{ - Faculty, HuginnMindRequest, HuginnMindResponse, ImportBatch, ImportFault, ImportPlan, PipelineAdmissionBatch, PipelineAdmissionOutcome, - PipelineDocumentSummary, PipelineDocumentView, PipelinePageItems, PipelineProvenance, PipelineSelectionPage, plan, + AdmissionFacts, Faculty, HuginnMindRequest, HuginnMindResponse, ImportBatch, ImportFault, ImportPlan, PipelineAdmissionBatch, PipelineAdmissionOutcome, + PipelineDocumentSummary, PipelineDocumentView, PipelinePageItems, PipelineProvenance, PipelineSelectionPage, inert_in_window, plan, }; use crate::config::{Settings, TOOL}; @@ -71,7 +71,12 @@ fn import(settings: &Settings, flags: Flags) -> Step<()> { let ask_receiver = |selection: Selection| { settings.ask(|instance| HuginnMindRequest::Query { instance: instance.clone(), selection, semantic: None }) }; - let held = resume(&plan, &headers(&ask_receiver, &campaign)?, giver.instance())?; + let root = headers(&ask_receiver, &campaign)?; + let held = resume(&plan, &root, giver.instance())?; + let window = window_writes(root.iter().map(|header| (header.id.kind, header.id.id.0.as_str(), &header.admission)), &planned_keys(&plan), giver.instance()); + if !window.is_empty() { + say(format_args!("left in place, admitted by the receiver after the import began: {}", window.iter().copied().collect::>().join(", "))); + } let (receipts, total) = (plan.batches.len(), plan.batches.iter().map(|batch| batch.documents.len()).sum::()); say(format_args!( "plan {campaign} from {giver}: {receipts} receipts {total} documents, {held} held, moved by {moved}", @@ -198,9 +203,27 @@ fn headers(ask: &dyn Fn(Selection) -> Result, campa Ok(headers) } +/// The receiver's own records under the root that were admitted after the first receipt imported +/// from this source and are no planned document: writes in the import window (ruling +/// `import-window-writes-ruled`: the import stops at a collision, and only a collision). They are +/// left where they are; they cannot be a planned document's key, and the admission of each later +/// batch still refuses any replay that meets one. +fn window_writes<'a>(root: impl Iterator + Clone, planned: &BTreeSet<&str>, source: &Slug) -> BTreeSet<&'a str> { + let began = root.clone().filter(|(_, _, admission)| admission.provenance.imported_from.as_ref().is_some_and(|from| from.instance == *source)).map(|(_, _, admission)| admission.ordinal).min(); + root.filter(|(kind, key, admission)| inert_in_window(*kind) && admission.provenance.imported_from.is_none() && !planned.contains(key) && began.is_some_and(|began| admission.ordinal > began)) + .map(|(_, key, _)| key) + .collect() +} + +/// The plan's document keys. +fn planned_keys(plan: &ImportPlan) -> BTreeSet<&str> { + plan.batches.iter().flat_map(|batch| batch.documents.iter().map(|(id, _)| id.id.0.as_str())).collect() +} + /// Which batches the receiver already holds whole. A document under the root that is not one of /// the plan's, imported from the plan's receipt of the source, is her own record and refuses the -/// import; a batch held in part cannot be resumed, only repaired. +/// import, unless she admitted it after the import began (`window_writes`); a batch held in part +/// cannot be resumed, only repaired. fn resume(plan: &ImportPlan, root: &[PipelineDocumentSummary], source: &Slug) -> Step> { let planned: BTreeMap<&str, (usize, &str)> = plan .batches @@ -210,8 +233,12 @@ fn resume(plan: &ImportPlan, root: &[PipelineDocumentSummary], source: &Slug) -> .collect(); let mut held: Vec> = vec![BTreeSet::new(); plan.batches.len()]; let mut own = Vec::new(); + let window = window_writes(root.iter().map(|header| (header.id.kind, header.id.id.0.as_str(), &header.admission)), &planned_keys(plan), source); for header in root { let key = header.id.id.0.as_str(); + if window.contains(key) { + continue; + } let imported = header.admission.provenance.imported_from.as_ref(); match planned.get(key) { Some((index, receipt)) if imported.is_some_and(|from| from.instance == *source && from.receipt_id.0 == *receipt) => { @@ -261,8 +288,9 @@ fn verify(root: &[PipelineDocumentView], plan: &ImportPlan, source: &Slug) -> St } } } - let planned: BTreeSet<&str> = plan.batches.iter().flat_map(|batch| batch.documents.iter().map(|(id, _)| id.id.0.as_str())).collect(); - differing.extend(held.keys().filter(|key| !planned.contains(*key)).map(|key| format!("{key}: not in the plan"))); + let planned = planned_keys(plan); + let window = window_writes(root.iter().map(|view| (view.id.kind, view.id.id.0.as_str(), &view.admission)), &planned, source); + differing.extend(held.keys().filter(|key| !planned.contains(*key) && !window.contains(*key)).map(|key| format!("{key}: not in the plan"))); if differing.is_empty() { Ok(()) } else { @@ -334,6 +362,16 @@ mod tests { } } + fn at(mut held: PipelineDocumentSummary, ordinal: u64) -> PipelineDocumentSummary { + held.admission.ordinal = ordinal; + held + } + + fn note(mut held: PipelineDocumentSummary, kind: PipelineKind) -> PipelineDocumentSummary { + held.id.kind = kind; + held + } + fn two() -> ImportPlan { plan_of(vec![batch(3, &["a", "b"]), batch(5, &["c"])]) } @@ -377,6 +415,62 @@ mod tests { } } + /// Nibu's case: her import stopped after its first receipt, her own run then admitted a + /// follow-up under the root, and the rerun was refused for it. + #[test] + fn a_record_the_receiver_admitted_after_the_import_began_does_not_block_the_resume() { + let source = slug("eureka"); + let root = [at(header("a", Some(("eureka", 3))), 10), at(header("b", Some(("eureka", 3))), 10), at(note(header("zz", None), PipelineKind::FollowUp), 20)]; + assert_eq!(resume(&two(), &root, &source).ok().unwrap(), vec![true, false]); + let window = window_writes(root.iter().map(|held| (held.id.kind, held.id.id.0.as_str(), &held.admission)), &planned_keys(&two()), &source); + assert_eq!(window.into_iter().collect::>(), vec!["c:ruling:zz"]); + } + + #[test] + fn only_a_native_unplanned_record_after_the_import_began_is_left_in_place() { + let source = slug("eureka"); + let began = [at(header("a", Some(("eureka", 3))), 10), at(header("b", Some(("eureka", 3))), 12)]; + for refused in [ + // admitted before the first imported receipt, however many came after + at(note(header("zz", None), PipelineKind::FollowUp), 9), + // a kind that could answer or resolve a planned document + at(header("zz", None), 20), + at(note(header("zz", None), PipelineKind::Resolution), 20), + // a planned key the receiver took first + at(note(header("c", None), PipelineKind::FollowUp), 20), + // another source's import, or this source's receipt for a document the plan lacks + at(note(header("zz", Some(("kiko", 3))), PipelineKind::FollowUp), 20), + at(note(header("zz", Some(("eureka", 9))), PipelineKind::FollowUp), 20), + ] { + let said = stopped(resume(&two(), &[began.as_slice(), &[refused]].concat(), &source)); + assert!(said.starts_with("no: Refused the receiver already holds records under the root"), "{said}"); + } + // the import is dated by its first receipt: a record between two imported ones is in the window + let between = [began.as_slice(), &[at(note(header("zz", None), PipelineKind::Idea), 11)]].concat(); + assert_eq!(resume(&two(), &between, &source).ok().unwrap(), vec![true, false]); + } + + #[test] + fn the_verification_walk_leaves_a_window_write_out_and_still_names_an_earlier_stranger() { + let source = slug("eureka"); + let plan = plan_of(vec![batch(3, &["a", "b"]), batch(5, &["c"])]); + let ours = |local: &str, ordinal, at: u64| { + let mut view = viewed(local, question(local), Some(("eureka", ordinal))); + view.admission.ordinal = at; + view + }; + let native = |local: &str, at: u64| { + let mut view = viewed(local, question(local), None); + view.id.kind = PipelineKind::FollowUp; + view.admission.ordinal = at; + view + }; + let whole = vec![ours("a", 3, 10), ours("b", 3, 10), ours("c", 5, 11)]; + assert!(verify(&[whole.as_slice(), &[native("zz", 20)]].concat(), &plan, &source).is_ok()); + let Err(Stop::Trouble(trouble)) = verify(&[whole.as_slice(), &[native("zz", 4)]].concat(), &plan, &source) else { panic!("a record before the import is not the window's") }; + assert!(trouble.to_string().contains("c:ruling:zz: not in the plan"), "{trouble}"); + } + fn viewed(local: &str, document: PipelineDocument, from: Option<(&str, u64)>) -> PipelineDocumentView { let held = header(local, from); PipelineDocumentView { id: held.id, document, admission: held.admission, status: PipelineStatus::InForce } diff --git a/crates/eureka-state/tests/cli.rs b/crates/eureka-state/tests/cli.rs index de4b2e0..0156131 100644 --- a/crates/eureka-state/tests/cli.rs +++ b/crates/eureka-state/tests/cli.rs @@ -1490,6 +1490,20 @@ fn question_of(campaign: &str, label: &str, title: &str) -> PipelineDocument { }) } +fn idea_of(campaign: &str, label: &str) -> PipelineDocument { + PipelineDocument::Idea(huginn_mind::eureka_pipeline::PipelineIdea { + campaign: slug(campaign), + pass: label.into(), + label: label.into(), + title: huginn_mind::eureka_pipeline::Title(label.into()), + idea: "A note.".into(), + tradeoffs: "".into(), + rests_on: vec![], + toward: vec![], + proposed_on: Date("2026-10-10".into()), + }) +} + fn for_lines(page: &str) -> Vec<&str> { page.lines().filter(|line| line.starts_with("For ")).collect() } @@ -2626,6 +2640,41 @@ fn import_resumes_after_a_stop() { assert!(imported_lines(&again.out).is_empty() && again.out.lines().next().unwrap().contains(&format!(", {} held,", plan.batches.len())), "{}", again.out); } +#[test] +fn import_resumes_past_a_record_the_receiver_admitted_after_it_began() { + let (root, mut giver_daemon) = giver_history(Some(TAKER)); + let plan = plan_from(&mut giver_daemon); + let giver = serve(root, giver_daemon); + let (root, mut daemon) = taker_mind(false); + half_imported(&mut daemon, &plan, 2, 0); + let own = answered(daemon.handle(HuginnMindRequest::Admit(batch(TAKER, vec![idea_of(CAMPAIGN, "Mine")])), chrono::Utc::now())); + assert!(matches!(own, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. }))); + let taker = serve(root, daemon); + + let ran = importing(&taker, INSTANCE, giver.addr); + assert_eq!(ran.code, 0, "{}{}", ran.out, ran.err); + assert!(ran.out.contains(&format!("left in place, admitted by the receiver after the import began: {CAMPAIGN}:idea:Mine.Mine")), "{}", ran.out); + assert_eq!(imported_lines(&ran.out).len(), plan.batches.len() - 2, "{}", ran.out); + assert!(ran.out.trim_end().ends_with(", verified"), "{}", ran.out); +} + +#[test] +fn import_still_refuses_a_native_question_admitted_after_it_began() { + let (root, mut giver_daemon) = giver_history(Some(TAKER)); + let plan = plan_from(&mut giver_daemon); + let giver = serve(root, giver_daemon); + let (root, mut daemon) = taker_mind(false); + half_imported(&mut daemon, &plan, 2, 0); + let own = answered(daemon.handle(HuginnMindRequest::Admit(batch(TAKER, vec![question_of(CAMPAIGN, "Mine", "Hers")])), chrono::Utc::now())); + assert!(matches!(own, HuginnMindResponse::Admit(PipelineAdmissionOutcome::Committed { .. }))); + let taker = serve(root, daemon); + + let ran = importing(&taker, INSTANCE, giver.addr); + assert_eq!(ran.code, 1, "{}{}", ran.out, ran.err); + assert!(ran.out.contains(&format!("{CAMPAIGN}:question:Mine")), "{}", ran.out); + assert!(imported_lines(&ran.out).is_empty(), "nothing is sent past a question the receiver wrote: {}", ran.out); +} + #[test] fn import_refuses_a_receipt_the_receiver_holds_only_part_of() { let (root, mut giver_daemon) = giver_history(Some(TAKER)); diff --git a/crates/huginn-mind/src/image.rs b/crates/huginn-mind/src/image.rs index 6f0b2ae..2f0ba7a 100644 --- a/crates/huginn-mind/src/image.rs +++ b/crates/huginn-mind/src/image.rs @@ -143,13 +143,17 @@ impl Image { self.closers.get(&kind).and_then(|named| named.get(id)).into_iter().flat_map(|keyed| keyed.iter().map(|(key, id)| (key.as_str(), *id))) } - /// Whether any document under `root` was admitted natively rather than imported. + /// Whether this mind holds a native write under `root` that an import must not meet: one admitted + /// before the first imported document of the root, or after it and not `inert_in_window`. pub(crate) fn root_holds_native_write(&self, root: &str) -> bool { - PipelineKind::ALL.iter().any(|kind| { - self.posting(*kind, Alias::Root.name(), root) - .into_iter() - .flatten() - .any(|id| self.row(*id).view.admission.provenance.imported_from.is_none()) + let rows: Vec<(PipelineKind, &SelectionRow)> = PipelineKind::ALL + .iter() + .flat_map(|kind| self.posting(*kind, Alias::Root.name(), root).into_iter().flatten().map(move |id| (*kind, self.row(*id)))) + .collect(); + let began = rows.iter().filter(|(_, row)| row.view.admission.provenance.imported_from.is_some()).map(|(_, row)| row.view.admission.ordinal).min(); + rows.iter().any(|(kind, row)| { + row.view.admission.provenance.imported_from.is_none() + && !(crate::import::inert_in_window(*kind) && began.is_some_and(|began| row.view.admission.ordinal > began)) }) } diff --git a/crates/huginn-mind/src/import.rs b/crates/huginn-mind/src/import.rs index 52b5335..19f9559 100644 --- a/crates/huginn-mind/src/import.rs +++ b/crates/huginn-mind/src/import.rs @@ -51,6 +51,15 @@ pub enum ImportFault { Unordered { ordinal: u64, id: Short }, } +/// Whether a native write of this kind, admitted after an import under its root began, cannot touch +/// the status or derivation of any document the import will still replay: a follow-up or an idea +/// has no referent a replay resolves and derives nothing. Ruling `import-window-writes-ruled` +/// stops an import at a collision; this is the writes that cannot collide. Every other kind may +/// answer, supersede or resolve a planned document, and still refuses the import. +pub fn inert_in_window(kind: PipelineKind) -> bool { + matches!(kind, PipelineKind::FollowUp | PipelineKind::Idea) +} + /// The plan for `receiver` to import `campaign` from `source`, given every document the source /// holds under the campaign's root. pub fn plan(campaign: &Slug, source: &Slug, receiver: &Slug, views: Vec) -> Result { diff --git a/crates/huginn-mind/src/lib.rs b/crates/huginn-mind/src/lib.rs index 47837d4..337778d 100644 --- a/crates/huginn-mind/src/lib.rs +++ b/crates/huginn-mind/src/lib.rs @@ -55,7 +55,7 @@ mod move_tests; pub use docs::CitationRole; pub use index::{INDEX_TEXT_MAX_BYTES, INDEX_TEXT_VERSION, IndexEntry, index_text}; pub use admission::{BATCH_MAX, PipelineAdmissionBatch, PipelineAdmissionOutcome}; -pub use import::{ImportBatch, ImportFault, ImportPlan, plan}; +pub use import::{ImportBatch, ImportFault, ImportPlan, inert_in_window, plan}; pub use mind::{HuginnMindEpoch, Mind}; pub use persona::{PersonaPutOutcome, PersonaStateView}; pub use query::{ diff --git a/crates/huginn-mind/src/move_tests.rs b/crates/huginn-mind/src/move_tests.rs index 37ef456..abcc64e 100644 --- a/crates/huginn-mind/src/move_tests.rs +++ b/crates/huginn-mind/src/move_tests.rs @@ -582,6 +582,37 @@ fn an_import_lands_only_under_a_root_that_only_imports_have_written() { ); } +#[test] +fn an_import_resumes_past_a_follow_up_or_idea_the_receiver_wrote_after_it_began() { + let q1 = id("question", "Q1"); + let begun = || { + let mut recv = receiver(); + committed(import_from_kiko(&mut recv, vec![campaign(&[REPO]), question("Q1", &["A", "B"], "A")])); + recv + }; + for native in [follow_up("FU-N", r(K::Question, &q1)), idea("p", "note", "a note")] { + let mut recv = begun(); + committed(admit(&mut recv, vec![native])); + committed(import_from_kiko(&mut recv, vec![question("Q2", &["A", "B"], "A")])); + committed(import_from_kiko(&mut recv, vec![answering(CAMPAIGN, "R1", &q1)])); + } + + // Before the first imported document the same record is the receiver's own, as ever. + let mut early = receiver(); + committed(admit(&mut early, vec![idea("p", "note", "a note")])); + assert_eq!( + refusal(import_from_kiko(&mut early, vec![campaign(&[REPO]), question("Q1", &["A", "B"], "A")])), + MindRefusal::ImportOutsideCampaign { root: CAMPAIGN.into() } + ); + + // A kind that could answer, supersede or resolve a replayed document still refuses, after the import began. + for native in [question("Q9", &["A", "B"], "A"), resolution(r(K::Question, &q1), ResolutionOutcome::Withdrawn { reason: "native".into() })] { + let mut recv = begun(); + committed(admit(&mut recv, vec![native])); + assert_eq!(refusal(import_from_kiko(&mut recv, vec![question("Q2", &["A", "B"], "A")])), MindRefusal::ImportOutsideCampaign { root: CAMPAIGN.into() }); + } +} + #[test] fn a_replay_under_another_source_is_no_replay() { let mut recv = receiver();