From a7b602ae6568ab255ea296d71f97877a2e7ea12e Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:48:33 +0200 Subject: [PATCH 001/264] WIP B3: route-proof readiness in the phase machine Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 571 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 457 insertions(+), 114 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index ac38edf..1977c9b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -262,20 +262,36 @@ impl RuntimePresenceEvidence { enum WarmingEvidence { OdinTopology { evidence: TopologyEvidence }, FirstOdinDirect { evidence: RuntimePresenceEvidence }, + /// A route-proof target's candidate answered Idunn's challenge itself. + RouteProofDirect { evidence: RuntimePresenceEvidence }, } impl WarmingEvidence { fn validate_shape(&self) -> Result<()> { match self { Self::OdinTopology { evidence } => evidence.validate_shape(), - Self::FirstOdinDirect { evidence } => evidence.validate_shape(), + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + evidence.validate_shape() + } } } fn canonical_sha256(&self) -> &str { match self { Self::OdinTopology { evidence } => &evidence.canonical_sha256, - Self::FirstOdinDirect { evidence } => &evidence.canonical_sha256, + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + &evidence.canonical_sha256 + } + } + } + + /// The presence Idunn challenged for itself, when no Odin carried it. + fn direct(&self) -> Option<&RuntimePresenceEvidence> { + match self { + Self::OdinTopology { .. } => None, + Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { + Some(evidence) + } } } } @@ -315,21 +331,24 @@ impl SequenceAdmittedWarming { }) } - fn from_first_odin_presence( + fn from_direct_presence( transaction_id: String, - evidence: RuntimePresenceEvidence, + warming: WarmingEvidence, present: cultnet_rs::VerifiedRuntimePresence, ) -> Result { + let evidence = warming + .direct() + .context("direct Warming evidence is an Odin topology receipt")?; ensure!( present.canonical_bytes() == evidence.canonical_bytes && present.signed_presence_sha256() == evidence.canonical_sha256, - "direct first-Odin Warming evidence differs from its authenticated presence" + "direct Warming evidence differs from its authenticated presence" ); let runtime_instance_id = present.record().runtime_instance_id.clone(); Ok(Self { transaction_id, signed_presence_sha256: evidence.canonical_sha256.clone(), - evidence: WarmingEvidence::FirstOdinDirect { evidence }, + evidence: warming, runtime_instance_id, }) } @@ -526,24 +545,58 @@ impl RoutingEvidence { } } +/// The states a route-proof candidate may answer its Warming challenge in. A +/// stateful one is warming until it holds a lease it cannot hold yet, and the +/// lease binds that warming presence. A stateless one may have finished +/// warming before the first challenge landed. +fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'static [&'static str] { + if expected.write_lease_required { + &["warming"] + } else { + &["warming", "active"] + } +} + +/// What a direct challenge to a candidate endpoint produced. +enum CandidateAnswer { + /// No answer: the reason it is still being waited on. + Silent(String), + Answered { + evidence: RuntimePresenceEvidence, + present: cultnet_rs::VerifiedRuntimePresence, + }, +} + /// Which kind of proof admits a target as Ready. Derived from the target's /// Expected, never stored: Expected is digest-bound, so the class cannot drift /// from the incarnation it describes. A target that declares a /// `shared-infrastructure odin.verse-rendezvous` dependency advertises into the -/// Verse and is Odin-correlated; one that declares none is route-proof. +/// Verse and is Odin-correlated; a routed one that declares none is +/// route-proof. Odin itself provides the rendezvous capability rather than +/// depending on it. It collects the same receipts an Odin-correlated target +/// does, and its admitted generation is what names the Odin authority, so it +/// is never route-proof. A target with no route has no endpoint to challenge, +/// so only Odin can report on it and it stays Odin-correlated. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ReadinessClass { + OdinSelf, OdinCorrelated, RouteProof, } impl ReadinessClass { fn of(expected: &IdunnExpectedIncarnationRecord) -> Self { + let provides_odin = expected + .capabilities + .iter() + .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY); let declares_odin = expected.dependencies.iter().any(|dependency| { dependency.kind == "shared-infrastructure" && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY }); - if declares_odin { + if provides_odin { + Self::OdinSelf + } else if declares_odin || expected.route.is_none() { Self::OdinCorrelated } else { Self::RouteProof @@ -1334,6 +1387,12 @@ impl DeploymentTransaction { self.target == "odin", "direct Warming evidence is reserved for Odin observing itself" ), + WarmingEvidence::RouteProofDirect { .. } => ensure!( + self.expected.as_ref().is_none_or(|expected| { + ReadinessClass::of(expected) == ReadinessClass::RouteProof + }), + "direct route-proof Warming evidence for a target Odin observes" + ), } } if let Some(leasing) = &self.leasing { @@ -1786,7 +1845,7 @@ impl AdmittedGeneration { fn from_transaction( transaction: &DeploymentTransaction, - odin_authority: AdmittedOdinAuthority, + odin_authority: Option, incumbent: Option<&AdmittedGeneration>, now: u64, ) -> Result { @@ -1795,15 +1854,15 @@ impl AdmittedGeneration { "only Committing can create an admitted generation" ); let ready = required(&transaction.ready, "Ready receipt")?.clone(); - let (latest_odin_observation, odin_authority) = match ready.class() { - ReadinessClass::OdinCorrelated => ( + let (latest_odin_observation, odin_authority) = match &ready { + ReadinessEvidence::OdinCorrelated { .. } => ( Some( required(&transaction.latest_odin_observation, "latest Odin receipt")? .clone(), ), - Some(odin_authority), + Some(required(&odin_authority, "Odin authority")?.clone()), ), - ReadinessClass::RouteProof => (None, None), + ReadinessEvidence::RouteProof { .. } => (None, None), }; let routing = required(&transaction.routing, "route disposition")?.clone(); let route_supervision = matches!(&routing, RoutingEvidence::Promoted { .. }).then(|| { @@ -3890,8 +3949,29 @@ impl Engine { &evidence.canonical_bytes, )?; } + WarmingEvidence::RouteProofDirect { evidence } => { + self.reauthenticate_route_proof( + transaction, + evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + evidence.admitted_at_unix_millis, + )?; + } } } + if let Some(ReadinessEvidence::RouteProof { evidence }) = &transaction.ready { + self.reauthenticate_route_proof( + transaction, + evidence, + &["active"], + lease, + evidence.admitted_at_unix_millis, + )?; + } if let Some(evidence) = transaction.ready.as_ref().and_then(ReadinessEvidence::odin) { let authenticated = self.authenticate_topology_bytes( snapshot, @@ -5253,6 +5333,30 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let incumbent_lease_sha256 = self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; + if current.value.warming.is_none() && ReadinessClass::of(expected) == ReadinessClass::RouteProof { + // A route-proof target's warming is Idunn's own challenge to the + // candidate endpoint. No Odin is read: the target declared no + // reason to be aware of one. + return match self.challenge_candidate( + ¤t.value, + route_proof_warming_states(expected), + None, + )? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { evidence, present } => { + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + self.persist_same_phase(current, |next| { + next.warming = Some(warming); + Ok(()) + }) + } + }; + } if current.value.warming.is_none() { // Odin's warming is observed directly when nothing else can observe // it. That is the first bootstrap, and it is also every continuity @@ -5287,13 +5391,14 @@ impl Engine { "direct Odin warming must be a stateful incarnation" ); let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let _token = SequenceAdmittedWarming::from_first_odin_presence( + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let _token = SequenceAdmittedWarming::from_direct_presence( current.value.transaction_id.clone(), - evidence.clone(), + warming.clone(), present, )?; return self.persist_same_phase(current, |next| { - next.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + next.warming = Some(warming); Ok(()) }); } @@ -5680,6 +5785,34 @@ impl Engine { "candidate process write lease is no longer exact" ); } + if ReadinessClass::of(expected) == ReadinessClass::RouteProof { + // Ready is the candidate answering Idunn's own challenge Active, + // holding the exact lease Idunn granted. Once recorded it is + // history; the stable route's challenge is what keeps proving it. + if matches!( + current.value.ready, + Some(ReadinessEvidence::RouteProof { .. }) + ) { + return self.transition(current, DeploymentPhase::Routing); + } + let lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + return match self.challenge_candidate(¤t.value, &["warming", "active"], lease)? { + CandidateAnswer::Silent(reason) => self.record_gate_wait(current, &reason), + CandidateAnswer::Answered { present, .. } if present.record().state == "warming" => { + self.record_gate_wait(current, "candidate is still warming") + } + CandidateAnswer::Answered { evidence, .. } => { + self.persist_same_phase(current, |next| { + next.ready = Some(ReadinessEvidence::RouteProof { evidence }); + Ok(()) + }) + } + }; + } { let current_lease = current .value @@ -5725,39 +5858,55 @@ impl Engine { let activation = required(¤t.value.activation, "activation")?; self.observe_candidate_before_waiting(¤t.value)?; if current.value.routing.is_none() { - let current_lease = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256); - let Some((admitted, authenticated)) = - self.admit_latest_topology(current, current_lease)? - else { - return Ok(()); + let admitted = if ReadinessClass::of(expected) == ReadinessClass::RouteProof { + // Ready is the candidate's own proof, recorded in AwaitingReady. + // No Odin correlation is read to admit a route. + ensure!( + matches!( + current.value.ready, + Some(ReadinessEvidence::RouteProof { .. }) + ), + "route-proof Routing without route-proof Ready evidence" + ); + current.clone() + } else { + let current_lease = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256); + let Some((admitted, authenticated)) = + self.admit_latest_topology(current, current_lease)? + else { + return Ok(()); + }; + if admitted.envelope != current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at route admission" + ); + let latest = required( + &admitted.value.latest_odin_observation, + "current route-admission topology evidence", + )?; + if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) + != Some(latest) + { + let latest = latest.clone(); + return self.persist_same_phase(&admitted, |next| { + next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); + Ok(()) + }); + } + let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; + ensure!( + ready.transaction_id() == admitted.value.transaction_id, + "Ready token belongs to another transaction" + ); + admitted }; - if admitted.envelope != current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at route admission" - ); - let latest = required( - &admitted.value.latest_odin_observation, - "current route-admission topology evidence", - )?; - if admitted.value.ready.as_ref().and_then(ReadinessEvidence::odin) != Some(latest) { - let latest = latest.clone(); - return self.persist_same_phase(&admitted, |next| { - next.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: latest }); - Ok(()) - }); - } - let ready = self.rehydrate_ready_token(&admitted.value, now_millis()?, true)?; - ensure!( - ready.transaction_id() == admitted.value.transaction_id, - "Ready token belongs to another transaction" - ); validate_live_providers_for_deploy(admitted.value.command_kind, || { self.validate_selected_providers_current(required( &admitted.value.plan, @@ -5834,32 +5983,50 @@ impl Engine { fn advance_committing(&self, current: &Stored) -> Result<()> { self.observe_candidate_before_waiting(¤t.value)?; - let current_lease_sha256 = current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - let Some((ready_current, authenticated)) = - self.admit_latest_topology(current, current_lease_sha256.as_deref())? - else { - return Ok(()); + let route_proof = ReadinessClass::of(required( + ¤t.value.expected, + "Expected projection", + )?) == ReadinessClass::RouteProof; + let ready_current = if route_proof { + // The candidate's Ready proof is durable, and the stable route's + // challenge below is its currency. No Odin is read to commit. + ensure!( + matches!( + current.value.ready, + Some(ReadinessEvidence::RouteProof { .. }) + ), + "route-proof commit without route-proof Ready evidence" + ); + current.clone() + } else { + let current_lease_sha256 = current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((ready_current, authenticated)) = + self.admit_latest_topology(current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if ready_current.envelope != current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready at admission commit" + ); + // Not require_current: the Ready receipt is durable evidence, so + // authenticating it against `now` asks a stored record to be live and + // refuses it once it ages past the 30s observation window. Route + // admission (:3906) refreshes the receipt to the latest observation and + // can therefore demand currency; by Committing the receipt is history. + // Currency here is answered by the latest observation, authenticated + // and checked for semantic readiness immediately above. + self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; + ready_current }; - if ready_current.envelope != current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready at admission commit" - ); - // Not require_current: the Ready receipt is durable evidence, so - // authenticating it against `now` asks a stored record to be live and - // refuses it once it ages past the 30s observation window. Route - // admission (:3906) refreshes the receipt to the latest observation and - // can therefore demand currency; by Committing the receipt is history. - // Currency here is answered by the latest observation, authenticated - // and checked for semantic readiness immediately above. - self.rehydrate_ready_token(&ready_current.value, now_millis()?, false)?; validate_live_providers_for_deploy(ready_current.value.command_kind, || { self.validate_selected_providers_current(required( &ready_current.value.plan, @@ -5909,26 +6076,31 @@ impl Engine { self.ensure_transaction_write_lease_current(&ready_current.value, now_millis()?)?; } - let current_lease_sha256 = ready_current - .value - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .map(str::to_owned); - let Some((commit_current, authenticated)) = - self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? - else { - return Ok(()); + let commit_current = if route_proof { + ready_current + } else { + let current_lease_sha256 = ready_current + .value + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .map(str::to_owned); + let Some((commit_current, authenticated)) = + self.admit_latest_topology(&ready_current, current_lease_sha256.as_deref())? + else { + return Ok(()); + }; + if commit_current.envelope != ready_current.envelope { + return Ok(()); + } + ensure!( + is_semantic_ready(&authenticated), + "latest Odin observation is not Ready after the final admission challenge" + ); + self.rehydrate_ready_token(&commit_current.value, now_millis()?, false)?; + commit_current }; - if commit_current.envelope != ready_current.envelope { - return Ok(()); - } - ensure!( - is_semantic_ready(&authenticated), - "latest Odin observation is not Ready after the final admission challenge" - ); let now = now_millis()?; - self.rehydrate_ready_token(&commit_current.value, now, false)?; validate_live_providers_for_deploy(commit_current.value.command_kind, || { self.validate_selected_providers_current(required( &commit_current.value.plan, @@ -5945,7 +6117,9 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let incumbent = self.exact_incumbent(&snapshot, &commit_current.value)?; - let odin_authority = self.current_odin_authority(&snapshot)?; + let odin_authority = (!route_proof) + .then(|| self.current_odin_authority(&snapshot)) + .transpose()?; let generation = AdmittedGeneration::from_transaction( &commit_current.value, odin_authority, @@ -6069,20 +6243,38 @@ impl Engine { "direct Warming refresh is reserved for Odin" ); let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; - let token = SequenceAdmittedWarming::from_first_odin_presence( + let warming = WarmingEvidence::FirstOdinDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( current.value.transaction_id.clone(), - evidence.clone(), + warming.clone(), present, )?; ensure!( token.signed_presence_sha256() != prior.signed_presence_sha256(), "first Odin replayed its pre-fence Warming presence" ); - Ok(Some(( - current.clone(), - WarmingEvidence::FirstOdinDirect { evidence }, - token, - ))) + Ok(Some((current.clone(), warming, token))) + } + WarmingEvidence::RouteProofDirect { .. } => { + // A stateful route-proof candidate is warming until it holds + // the lease this refresh is about to bind, so the fresh + // presence must be warming again, and must be a new one. + let CandidateAnswer::Answered { evidence, present } = + self.challenge_candidate(¤t.value, &["warming"], None)? + else { + return Ok(None); + }; + let warming = WarmingEvidence::RouteProofDirect { evidence }; + let token = SequenceAdmittedWarming::from_direct_presence( + current.value.transaction_id.clone(), + warming.clone(), + present, + )?; + ensure!( + token.signed_presence_sha256() != prior.signed_presence_sha256(), + "route-proof candidate replayed its pre-fence Warming presence" + ); + Ok(Some((current.clone(), warming, token))) } WarmingEvidence::OdinTopology { evidence: prior_evidence, @@ -6421,6 +6613,7 @@ impl Engine { ) } + /// The stable route's proof: a challenged presence that is Active. fn authenticate_routed_presence( &self, authority: &cultnet_rs::VerifiedRuntimeAuthority, @@ -6430,9 +6623,41 @@ impl Engine { received_at_unix_millis: u64, canonical_presence: &[u8], ) -> Result<(String, u64)> { + let present = self.authenticate_challenged_presence( + authority, + &["active"], + current_write_lease_sha256, + message_id, + challenged_at_unix_millis, + received_at_unix_millis, + canonical_presence, + )?; + Ok(( + present.signed_presence_sha256().to_owned(), + received_at_unix_millis, + )) + } + + /// A presence Idunn challenged for itself, from the stable endpoint or the + /// candidate's: signed by the provider and the launch's activation key, + /// bound to current authority, minted after the challenge, answering that + /// exact challenge, in one of `states`. A warming presence holds no write + /// lease; a presence in any other state holds exactly the current one. + /// Every disagreement with authority is named, so a capacity below the + /// Expected minimum reads as that and not as a generic refusal. + fn authenticate_challenged_presence( + &self, + authority: &cultnet_rs::VerifiedRuntimeAuthority, + states: &[&str], + current_write_lease_sha256: Option<&str>, + message_id: &str, + challenged_at_unix_millis: u64, + received_at_unix_millis: u64, + canonical_presence: &[u8], + ) -> Result { ensure!( received_at_unix_millis >= challenged_at_unix_millis, - "route observation predates its challenge" + "route proof predates its challenge" ); let authenticated = authenticate_runtime_presence_claim( canonical_presence, @@ -6443,31 +6668,49 @@ impl Engine { maximum_future_skew_millis: self.options.topology_maximum_future_skew_millis, }, )?; - let signed_presence_sha256 = authenticated.signed_presence_sha256().to_owned(); let correlation = correlate_runtime_presence_claim(authenticated, authority)?; - ensure!( - correlation.disagreements().is_empty(), - "stable route answered with a runtime that disagrees with current authority" - ); + if !correlation.disagreements().is_empty() { + let named = correlation + .disagreements() + .iter() + .map(|disagreement| { + format!( + "{} (expected {}, observed {})", + disagreement.code, + disagreement.expected.as_deref().unwrap_or("none"), + disagreement.observed.as_deref().unwrap_or("none"), + ) + }) + .collect::>() + .join("; "); + bail!("route proof answered with a runtime that disagrees with current authority: {named}"); + } let present = correlation.into_undisputed_present()?; let presence = present.record(); ensure!( presence.observed_at_unix_millis >= challenged_at_unix_millis, - "stable route returned a presence minted before the route challenge" + "route proof returned a presence minted before its challenge" ); ensure!( - presence.state == "active", - "stable route runtime is not Active" + states.contains(&presence.state.as_str()), + "route proof runtime is {}, not {}", + presence.state, + states.join(" or ") ); ensure!( presence.detail == format!("route-observation:{message_id}"), - "stable route response is not bound to the exact challenge" + "route proof response is not bound to the exact challenge" ); + let lease_is_current = if presence.state == "warming" { + presence.write_lease_sha256.is_none() + } else { + presence.write_lease_sha256.as_deref() == current_write_lease_sha256 + }; ensure!( - presence.write_lease_sha256.as_deref() == current_write_lease_sha256, - "stable route runtime does not hold the exact current process write lease" + lease_is_current, + "route proof runtime does not hold the exact current process write lease" ); - Ok((signed_presence_sha256, received_at_unix_millis)) + Ok(present) } fn authenticate_first_odin_warming_presence( @@ -6800,15 +7043,110 @@ impl Engine { }, &evidence.canonical_bytes, )?; - SequenceAdmittedWarming::from_first_odin_presence( + SequenceAdmittedWarming::from_direct_presence( transaction.transaction_id.clone(), - evidence, + WarmingEvidence::FirstOdinDirect { evidence }, + present, + ) + } + WarmingEvidence::RouteProofDirect { evidence } => { + let present = self.reauthenticate_route_proof( + transaction, + &evidence, + route_proof_warming_states(required( + &transaction.expected, + "Warming Expected projection", + )?), + None, + if require_current { + now + } else { + evidence.admitted_at_unix_millis + }, + )?; + SequenceAdmittedWarming::from_direct_presence( + transaction.transaction_id.clone(), + WarmingEvidence::RouteProofDirect { evidence }, present, ) } } } + /// A route-proof presence Idunn recorded, authenticated again as of `at`. + fn reauthenticate_route_proof( + &self, + transaction: &DeploymentTransaction, + evidence: &RuntimePresenceEvidence, + states: &[&str], + current_write_lease_sha256: Option<&str>, + at_unix_millis: u64, + ) -> Result { + let authority = self.runtime_authority(transaction)?; + self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &evidence.message_id, + evidence.challenged_at_unix_millis, + at_unix_millis, + &evidence.canonical_bytes, + ) + } + + /// One direct challenge to a route-proof target's candidate endpoint. + /// Failing to reach or hear the candidate is only silence: a process that + /// is still starting does not answer, and that is waiting, not a fault. + /// What does answer is authenticated, and a bad answer is an error. + fn challenge_candidate( + &self, + transaction: &DeploymentTransaction, + states: &[&str], + current_write_lease_sha256: Option<&str>, + ) -> Result { + let expected = required(&transaction.expected, "candidate Expected")?; + let binding = required(&transaction.plan, "candidate plan")? + .parsed_inputs()? + .1; + let driver = NginxRouteDriver::new( + binding + .route + .context("route-proof candidate has no route binding")?, + ); + let message_id = format!("candidate-{}", Uuid::new_v4().simple()); + let challenged_at_unix_millis = now_millis()?; + let response = match driver.request_candidate_runtime_presence(expected, &message_id) { + Ok(response) => response, + Err(error) => { + return Ok(CandidateAnswer::Silent(format!( + "candidate endpoint did not answer its challenge: {error:#}" + ))); + } + }; + ensure!( + response.message_id == message_id, + "candidate transport substituted its challenge identity" + ); + let admitted_at_unix_millis = now_millis()?; + let authority = self.runtime_authority(transaction)?; + let present = self.authenticate_challenged_presence( + &authority, + states, + current_write_lease_sha256, + &message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + &response.canonical_presence, + )?; + let evidence = RuntimePresenceEvidence::from_present( + &present, + message_id, + challenged_at_unix_millis, + admitted_at_unix_millis, + )?; + Ok(CandidateAnswer::Answered { evidence, present }) + } + fn rehydrate_ready_token( &self, transaction: &DeploymentTransaction, @@ -6854,6 +7192,11 @@ impl Engine { ) -> Result> { let mut providers = Vec::new(); for stored in &snapshot.admitted { + // A route-proof generation holds no Odin receipt to hand a + // dependent; its currency is the route observation, which is B4's. + if stored.value.ready.class() == ReadinessClass::RouteProof { + continue; + } // No pre-filter on ready == latest: a provider that published again // after going ready is still ready. rehydrate_admitted_ready owns // that judgment now, and reports why when it refuses. From cfe71f680caa6ec17f3eb54f8d571eacb3daef4f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:54:22 +0200 Subject: [PATCH 002/264] WIP B3: route-proof harness and tests Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 670 +++++++++++++++++++++++++++++++++++++++++-- src/drivers.rs | 2 +- 2 files changed, 648 insertions(+), 24 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 1977c9b..7f30f47 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -10190,6 +10190,9 @@ mod tests { assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); expected.dependencies.clear(); assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + // Nothing to challenge: only Odin can report on an unrouted target. + expected.route = None; + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinCorrelated); Ok(()) } @@ -10274,7 +10277,7 @@ mod tests { .context("fixture generation has no Odin authority")?; let next = AdmittedGeneration::from_transaction( &committing, - authority.clone(), + Some(authority.clone()), Some(&incumbent), 1_700_000_200_000, )?; @@ -10287,7 +10290,7 @@ mod tests { let first = AdmittedGeneration::from_transaction( &committing, - authority, + Some(authority), None, 1_700_000_200_000, )?; @@ -11077,27 +11080,7 @@ mod tests { /// A route-proof generation from a real admission: the world's Expected /// declares no Odin dependency, so route proof is its class. fn route_proof_generation() -> Result { - let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - odin_reports_ready(&world, &seeded, 5)?; - drive(&world, |transaction| { - transaction.phase == DeploymentPhase::Committing - })?; - let mut committing = resident(&world)?.value; - assert_eq!( - ReadinessClass::of(committing.expected.as_ref().unwrap()), - ReadinessClass::RouteProof - ); - committing.ready = Some(route_proof_evidence()); - committing.latest_odin_observation = None; - let generation = AdmittedGeneration::from_transaction( - &committing, - world.engine.bootstrap_odin_authority.clone(), - None, - now_millis()?, - )?; - generation.validate()?; - Ok(generation) + route_proof::committed_generation() } #[test] @@ -13265,4 +13248,645 @@ mod tests { assert!(ControlSnapshot::read(&world.state_store)?.transactions.is_empty()); Ok(()) } + + // --------------------------------------------------------------------- + // Route-proof readiness: a routed target that declares no Odin dependency + // is admitted by Idunn's own challenges. Odin is never read. + // --------------------------------------------------------------------- + mod route_proof { + use super::*; + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; + use std::time::Duration; + + /// The runtime behind both endpoints, answering Idunn's snapshot + /// challenge the way the TypeScript signer does: locally, in the + /// reported health, with `route-observation:` as the detail. + struct RuntimeStub { + expected: IdunnExpectedIncarnationRecord, + activation: IdunnRuntimeActivationRecord, + provider: cultnet_rs::ServiceIdentitySigner, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, + state: Mutex<&'static str>, + capacity: AtomicU32, + sequence: AtomicU64, + candidate_hits: AtomicUsize, + stable_hits: AtomicUsize, + stop: AtomicBool, + } + + impl RuntimeStub { + fn answer(&self, message_id: &str) -> Result> { + let expected = &self.expected; + let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { + schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + target: expected.target.clone(), + expected_projection_sha256: self.activation.expected_projection_sha256.clone(), + plan_id: expected.plan_id.clone(), + incarnation_id: expected.incarnation_id.clone(), + sealed_release_id: expected.sealed_release_id.clone(), + activation_witness_sha256: self.activation.canonical_sha256()?, + state_schema_generation: expected.state_schema_generation.clone(), + state_contract_sha256: expected.state_contract_sha256.clone(), + runtime_id: expected.runtime_id.clone(), + runtime_instance_id: self.activation.runtime_instance_id.clone(), + bound_endpoint: expected + .route + .as_ref() + .map(|route| route.candidate_endpoint.clone()), + capabilities: expected + .capabilities + .iter() + .map(|capability| cultnet_rs::GameCultRuntimeCapability { + capability: capability.capability.clone(), + schema: capability.schema.clone(), + compatibility: capability.compatibility.clone(), + capacity: self.capacity.load(Ordering::SeqCst), + }) + .collect(), + health_contract: expected.health_contract.clone(), + state: (*self.state.lock().unwrap()).into(), + detail: format!("route-observation:{message_id}"), + write_lease_sha256: None, + signer_identity_id: self.provider.entry().identity_id.clone(), + publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, + observed_at_unix_millis: now_millis()?, + signature_algorithm: "ed25519".into(), + signature: Vec::new(), + activation_signer_identity_id: self.activation_signer.identity_id(), + activation_signature: Vec::new(), + }; + let proof = presence.canonical_proof_payload()?; + presence.signature = self + .provider + .sign::(&proof) + .signature; + presence.activation_signature = + self.activation_signer.sign_presence_proof(&presence)?; + Ok(rmp_serde::to_vec(&presence)?) + } + + fn set_state(&self, state: &'static str) { + *self.state.lock().unwrap() = state; + } + + fn serve_one(&self, mut stream: TcpStream) -> Result<()> { + stream.set_read_timeout(Some(Duration::from_secs(3)))?; + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + let (header_end, length) = loop { + let read = stream.read(&mut chunk)?; + ensure!(read > 0, "stub client closed early"); + request.extend_from_slice(&chunk[..read]); + if let Some(index) = request.windows(4).position(|w| w == b"\r\n\r\n") { + let head = String::from_utf8_lossy(&request[..index]).to_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .context("stub request has no Content-Length")? + .trim() + .parse::()?; + if request.len() >= index + 4 + length { + break (index + 4, length); + } + } + }; + let message = cultnet_rs::decode_cultnet_message_from_slice( + &request[header_end..header_end + length], + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + let cultnet_rs::CultNetMessage::SnapshotRequest { message_id, .. } = message + else { + bail!("stub received something other than a snapshot request"); + }; + let response = cultnet_rs::CultNetMessage::SnapshotResponseRaw { + message_id: message_id.clone(), + documents: vec![cultnet_rs::CultNetRawDocumentRecord { + schema_id: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), + record_key: self.expected.target.clone(), + stored_at: rfc3339_millis(now_millis()?)?, + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload: self.answer(&message_id)?, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }], + }; + let body = cultnet_rs::encode_cultnet_message_to_vec( + &response, + cultnet_rs::CultNetWireContract::CultNetSchemaV0, + )?; + stream.write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + )?; + stream.write_all(&body)?; + Ok(()) + } + + fn listen(self: &Arc, listener: TcpListener, candidate: bool) { + listener.set_nonblocking(true).expect("nonblocking listener"); + let stub = Arc::clone(self); + std::thread::spawn(move || { + while !stub.stop.load(Ordering::SeqCst) { + match listener.accept() { + Ok((stream, _)) => { + stream.set_nonblocking(false).ok(); + let hits = if candidate { + &stub.candidate_hits + } else { + &stub.stable_hits + }; + hits.fetch_add(1, Ordering::SeqCst); + let _ = stub.serve_one(stream); + } + Err(_) => std::thread::sleep(Duration::from_millis(5)), + } + } + }); + } + } + + impl Drop for RuntimeStub { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + } + } + + /// Odin, as the world it runs in sees it. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Odin { + /// Every read of the correlation store fails. + Unreachable, + /// Odin publishes a correlation that says the target is not Ready. + NotReady, + } + + struct RoutedWorld { + world: EngineFixture, + stub: Arc, + odin: Odin, + poisoned: Vec, + } + + /// A stateless, routed target that declares no Odin dependency, over + /// `minimum_capacity`, seeded at `phase` (Warming or Fencing): its + /// Warming evidence, when the phase needs it, is the candidate's own + /// answer to a real challenge. Listeners stand at both endpoints. + fn routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + ) -> Result { + use crate::deployment_plan::tests::{ + BINDING, RECIPE, artifact_receipt, external_input_receipt, source, + }; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let candidate_listener = TcpListener::bind("127.0.0.1:0")?; + let stable_listener = TcpListener::bind("127.0.0.1:0")?; + let candidate_port = candidate_listener.local_addr()?.port(); + let stable_port = stable_listener.local_addr()?.port(); + + let provider_path = world.root.join("identities/provider.cc"); + let provider_anchor = world.root.join("identities/provider-anchor.cc"); + let provider = + enroll_service_identity_at::(&provider_path)?; + export_service_identity_trust_anchor(&provider, &provider_anchor)?; + let config = world.root.join("service.conf"); + let binding = BINDING + .replace( + "/etc/gamecult/trust/service.cc", + &provider_anchor.display().to_string(), + ) + .replace("service-runtime-signer", &provider.entry().identity_id) + .replace("127.0.0.1:17999", &format!("127.0.0.1:{stable_port}")) + .replace( + "private_port_start = 18000", + &format!("private_port_start = {candidate_port}"), + ) + .replace( + "private_port_end = 18009", + &format!("private_port_end = {candidate_port}"), + ) + .replace( + "/etc/nginx/idunn-stream-routes/service.conf", + &config.display().to_string().replace('\\', "/"), + ); + let recipe = RECIPE + .split("[[dependencies]]") + .next() + .context("recipe is empty")? + .replace( + "capability = \"service.runtime\"", + &format!("capability = \"service.runtime\"\ncapacity = {minimum_capacity}"), + ); + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + "service-incarnation-1", + Some(candidate_port), + 110, + &[], + )?; + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + )?; + let expected = release.expected_projection(&plan)?; + assert!(expected.route.is_some() && expected.dependencies.is_empty()); + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "continuity-service".into(), + kind: CommandKind::Continuity, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let mut transaction = + DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + transaction.lifecycle_authorized_at_unix_millis = Some(now); + let launch = IdunnRuntimeActivationLaunch::issue( + &expected, + runtime_instance_id(&transaction.transaction_id)?, + now, + &world.engine.idunn_signer, + )?; + let mut seed = Vec::new(); + let activation = launch.write_credential(&mut seed)?; + let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; + let mut workload = recorded.workload.clone().context("no recorded workload")?; + match &mut workload { + WorkloadObservation::Systemd(observed) => { + observed.runtime_instance_id = activation.runtime_instance_id.clone(); + observed.executable_sha256 = expected.artifact_sha256.clone(); + } + WorkloadObservation::Host(_) => bail!("recorded workload is not a systemd unit"), + } + transaction.isolation = recorded.isolation; + transaction.installed_release = Some(crate::drivers::InstalledReleaseObservation { + sealed_release_id: release.sealed_release_id.clone(), + root: PathBuf::from("/srv/service/releases/test"), + }); + transaction.expected_publication_sha256 = Some(expected.canonical_sha256()?); + transaction.activation_publication_sha256 = Some(activation.canonical_sha256()?); + transaction.sealed_release = Some(release); + transaction.expected = Some(expected.clone()); + transaction.activation = Some(activation.clone()); + transaction.workload = Some(workload); + transaction.plan = Some(plan); + + let stub = Arc::new(RuntimeStub { + expected, + activation, + provider: open_service_identity_at::( + &provider_path, + )?, + activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( + &seed[..], + )?, + state: Mutex::new("warming"), + capacity: AtomicU32::new(minimum_capacity), + sequence: AtomicU64::new(0), + candidate_hits: AtomicUsize::new(0), + stable_hits: AtomicUsize::new(0), + stop: AtomicBool::new(false), + }); + stub.listen(candidate_listener, true); + stub.listen(stable_listener, false); + + if phase >= DeploymentPhase::Fencing { + let CandidateAnswer::Answered { evidence, .. } = + world + .engine + .challenge_candidate(&transaction, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + let rendered = NginxRouteDriver::new( + transaction + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .route + .context("no route binding")?, + ) + .render(transaction.expected.as_ref().unwrap())?; + transaction.route_preflight = Some(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: transaction + .activation + .as_ref() + .unwrap() + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(&rendered), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }); + } + transaction.enter_phase(phase, now); + transaction.validate()?; + let store = SingleFileMessagePackBackingStore::new(&world.state_store); + assert!(store.compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )?); + + // Odin's world. Unreachable: the correlation store is bytes no + // reader can decode, so any read of it is an error. NotReady: a + // valid correlation that says the target is not Ready. + let store = &world.engine.options.odin_correlation_store; + let poisoned = match odin { + Odin::Unreachable => { + let garbage = b"odin is not a cache".to_vec(); + std::fs::write(store, &garbage)?; + garbage + } + Odin::NotReady => { + let wrongly = signed_correlation(&world, &transaction, 9, false)?; + SingleFileMessagePackBackingStore::new(store).insert_entry_if_absent( + CultCacheEnvelope { + key: crate::drivers::incarnation_key_of( + &transaction.target, + &transaction.expected.as_ref().unwrap().canonical_sha256()?, + ), + r#type: OdinRuntimeTopologyCorrelationRecord::TYPE.into(), + payload: wrongly, + stored_at: rfc3339_millis(now_millis()?)?, + schema_id: Some( + cultnet_rs::ODIN_RUNTIME_TOPOLOGY_CORRELATION_SCHEMA.into(), + ), + }, + )?; + Vec::new() + } + }; + Ok(RoutedWorld { + world, + stub, + odin, + poisoned, + }) + } + + impl RoutedWorld { + fn step(&self) -> Result<()> { + self.world.engine.advance_transaction(&resident(&self.world)?) + } + + fn run_to_routing(&self) -> Result<()> { + for _ in 0..12 { + if self.transaction()?.phase == DeploymentPhase::Routing { + return Ok(()); + } + self.step()?; + } + bail!("the transaction never reached Routing") + } + + fn transaction(&self) -> Result { + Ok(resident(&self.world)?.value) + } + + /// Odin was never read: the poisoned store is byte for byte what + /// it was, and no step failed on it. + fn assert_odin_untouched(&self) -> Result<()> { + if self.odin == Odin::Unreachable { + assert_eq!( + std::fs::read(&self.world.engine.options.odin_correlation_store)?, + self.poisoned + ); + } + Ok(()) + } + + /// Routing installs an nginx fragment and needs the host's programs, + /// so the world writes the fragment itself and records the promotion + /// the stable challenge proves, then enters Committing. + fn promote_by_hand(&self) -> Result<()> { + let current = resident(&self.world)?; + let transaction = ¤t.value; + assert_eq!(transaction.phase, DeploymentPhase::Routing); + let expected = transaction.expected.as_ref().unwrap(); + let binding = transaction.plan.as_ref().unwrap().parsed_inputs()?.1; + let driver = NginxRouteDriver::new(binding.route.unwrap()); + let rendered = driver.render(expected)?; + std::fs::write(&driver.binding.config_path, &rendered)?; + let observation = self.world.engine.prove_stable_route( + transaction, + &driver, + sha256_id(&rendered), + )?; + let mut next = transaction.clone(); + next.routing = Some(RoutingEvidence::Promoted { + promoted_at_unix_millis: observation.observed_at_unix_millis, + observation, + }); + next.enter_phase(DeploymentPhase::Committing, now_millis()?); + replace_transaction(&self.world.state_store, ¤t, &next) + } + } + + /// A route-proof generation from a real admission. + pub(super) fn committed_generation() -> Result { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.step()?; + admitted(&routed) + } + + fn admitted(world: &RoutedWorld) -> Result { + Ok(ControlSnapshot::read(&world.world.state_store)? + .admitted + .into_iter() + .next() + .context("nothing was admitted")? + .value) + } + + #[test] + fn a_route_proof_target_admits_with_odin_unreachable_and_never_reads_it() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + // Fencing, Leasing, AwaitingReady: the candidate is still warming. + for _ in 0..12 { + routed.step()?; + if routed.transaction()?.last_error.is_some() { + break; + } + } + let waiting = routed.transaction()?; + assert_eq!(waiting.phase, DeploymentPhase::AwaitingReady); + assert!(waiting.ready.is_none()); + assert_eq!( + waiting.last_error.as_deref(), + Some("candidate is still warming") + ); + // The candidate finishes warming: Ready is its own Active answer. + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.step()?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Routing); + routed.promote_by_hand()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert!(generation.odin_authority.is_none()); + assert!(generation.latest_odin_observation.is_none()); + assert!(generation.route_supervision.is_some()); + // Idunn's own challenges carried every proof: the seed's warming, + // two Ready polls; the stable endpoint proved the route twice. + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 3); + assert!(routed.stub.stable_hits.load(Ordering::SeqCst) >= 2); + routed.assert_odin_untouched()?; + + // Boot and supervision read no Odin either: the topology refresh + // returns before it opens Odin's store, and durable authority + // skips a route-proof generation's receipts. + let stored = ControlSnapshot::read(&routed.world.state_store)?; + let current = stored.admitted.first().context("no admitted generation")?; + assert!(!routed.world.engine.refresh_admitted_topology(&stored, current)?); + routed.world.engine.validate_durable_authority(&stored)?; + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn an_odin_correlation_about_a_route_proof_target_decides_nothing() -> Result<()> { + let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.step()?; + let generation = admitted(&routed)?; + assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + // The valid, not-Ready correlation was neither admitted as a + // receipt nor consulted: the record carries no Odin cursor. + assert_eq!(generation.odin_publisher_sequence_cursor, 0); + Ok(()) + } + + #[test] + fn a_route_proof_candidate_that_has_not_answered_is_waited_on_not_aborted() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + routed.step()?; + let warmed = routed.transaction()?; + assert_eq!(warmed.phase, DeploymentPhase::Warming); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &warmed.warming else { + bail!("Warming evidence is not the candidate's own presence"); + }; + assert!(evidence.message_id.starts_with("candidate-")); + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + routed.assert_odin_untouched()?; + + // Nothing listens: the same step waits, and says why. + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + silent.stub.stop.store(true, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(50)); + silent.step()?; + let waiting = silent.transaction()?; + assert!(waiting.warming.is_none() && waiting.pre_fencing_abort.is_none()); + assert!( + waiting + .last_error + .as_deref() + .is_some_and(|error| error.contains("candidate endpoint did not answer")), + "{:?}", + waiting.last_error + ); + Ok(()) + } + + #[test] + fn a_capacity_below_the_expected_minimum_is_named_on_the_route_path() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.stub.capacity.store(1, Ordering::SeqCst); + let failure = format!("{:#}", routed.step().expect_err("a shortfall must be refused")); + assert!( + failure.contains("expected-capability-000-capacity"), + "{failure}" + ); + assert!(admitted(&routed).is_err()); + Ok(()) + } + + #[test] + fn odin_itself_is_never_route_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + let mut expected = routed.stub.expected.clone(); + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + // Odin provides the rendezvous capability and declares no dependency. + expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + assert!(expected.dependencies.is_empty()); + assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinSelf); + + // Route-proof evidence for it is refused: its admitted generation + // is what names the Odin authority, so it must carry one. + let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; + odin.target = "odin".into(); + odin.expected = expected; + odin.ready = route_proof_evidence(); + odin.latest_odin_observation = None; + odin.odin_authority = None; + assert!(error_text(odin.validate()).contains("declares Odin")); + Ok(()) + } + + #[test] + fn a_store_of_route_proof_generations_boots_and_resolves_the_bootstrap_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + routed.promote_by_hand()?; + routed.step()?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + assert_eq!( + routed.world.engine.current_odin_authority(&snapshot)?, + routed.world.engine.bootstrap_odin_authority + ); + Ok(()) + } + } } diff --git a/src/drivers.rs b/src/drivers.rs index d56e3d9..44e4223 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5128,7 +5128,7 @@ impl NginxRouteDriver { Ok(output) } - fn render(&self, expected: &IdunnExpectedIncarnationRecord) -> Result> { + pub(crate) fn render(&self, expected: &IdunnExpectedIncarnationRecord) -> Result> { expected.validate()?; let expected_projection_sha256 = expected.canonical_sha256()?; let route = expected From 66fda86d8fe9a37ec615d48a2f222859cc905eb2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 10:56:26 +0200 Subject: [PATCH 003/264] WIP B3: two private ports in the routed world --- src/control_plane.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 7f30f47..a962ec9 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13472,7 +13472,7 @@ mod tests { ) .replace( "private_port_end = 18009", - &format!("private_port_end = {candidate_port}"), + &format!("private_port_end = {}", candidate_port + 1), ) .replace( "/etc/nginx/idunn-stream-routes/service.conf", From b9864b5041a4314b5f78f51f402f094bb498b515 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 11:00:47 +0200 Subject: [PATCH 004/264] WIP B3: Odin-providing target is driven through the engine --- src/control_plane.rs | 82 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 11 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index a962ec9..712dc4a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13443,6 +13443,7 @@ mod tests { odin: Odin, minimum_capacity: u32, phase: DeploymentPhase, + provides_odin: bool, ) -> Result { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, @@ -13484,7 +13485,14 @@ mod tests { .context("recipe is empty")? .replace( "capability = \"service.runtime\"", - &format!("capability = \"service.runtime\"\ncapacity = {minimum_capacity}"), + &format!( + "capability = \"{}\"\ncapacity = {minimum_capacity}", + if provides_odin { + ODIN_RENDEZVOUS_CAPABILITY + } else { + "service.runtime" + } + ), ); let plan = compile_deployment_plan( recipe.as_bytes(), @@ -13503,7 +13511,14 @@ mod tests { )?; let expected = release.expected_projection(&plan)?; assert!(expected.route.is_some() && expected.dependencies.is_empty()); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + assert_eq!( + ReadinessClass::of(&expected), + if provides_odin { + ReadinessClass::OdinSelf + } else { + ReadinessClass::RouteProof + } + ); let now = now_millis()?; let command = DeploymentCommand { @@ -13566,7 +13581,22 @@ mod tests { stub.listen(candidate_listener, true); stub.listen(stable_listener, false); - if phase >= DeploymentPhase::Fencing { + if phase >= DeploymentPhase::Fencing && provides_odin { + // Odin's own first word about the candidate, as for any + // Odin-correlated target. + let warming = signed_correlation(&world, &transaction, 4, false)?; + let authenticated = world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&world.state_store)?, + &transaction, + &warming, + None, + now, + )?; + transaction.warming = Some(WarmingEvidence::OdinTopology { + evidence: TopologyEvidence::from_authenticated(&authenticated, now)?, + }); + transaction.odin_publisher_sequence_cursor = 4; + } else if phase >= DeploymentPhase::Fencing { let CandidateAnswer::Answered { evidence, .. } = world .engine @@ -13575,6 +13605,8 @@ mod tests { bail!("the stub candidate did not answer"); }; transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + } + if phase >= DeploymentPhase::Fencing { let rendered = NginxRouteDriver::new( transaction .plan @@ -13627,6 +13659,10 @@ mod tests { // valid correlation that says the target is not Ready. let store = &world.engine.options.odin_correlation_store; let poisoned = match odin { + _ if provides_odin => { + odin_reports_ready(&world, &transaction, 5)?; + Vec::new() + } Odin::Unreachable => { let garbage = b"odin is not a cache".to_vec(); std::fs::write(store, &garbage)?; @@ -13719,7 +13755,7 @@ mod tests { /// A route-proof generation from a real admission. pub(super) fn committed_generation() -> Result { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.promote_by_hand()?; @@ -13738,7 +13774,7 @@ mod tests { #[test] fn a_route_proof_target_admits_with_odin_unreachable_and_never_reads_it() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; // Fencing, Leasing, AwaitingReady: the candidate is still warming. for _ in 0..12 { routed.step()?; @@ -13790,7 +13826,7 @@ mod tests { #[test] fn an_odin_correlation_about_a_route_proof_target_decides_nothing() -> Result<()> { - let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::NotReady, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.promote_by_hand()?; @@ -13805,7 +13841,7 @@ mod tests { #[test] fn a_route_proof_candidate_that_has_not_answered_is_waited_on_not_aborted() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; routed.step()?; let warmed = routed.transaction()?; assert_eq!(warmed.phase, DeploymentPhase::Warming); @@ -13817,7 +13853,7 @@ mod tests { routed.assert_odin_untouched()?; // Nothing listens: the same step waits, and says why. - let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; silent.stub.stop.store(true, Ordering::SeqCst); std::thread::sleep(Duration::from_millis(50)); silent.step()?; @@ -13836,7 +13872,7 @@ mod tests { #[test] fn a_capacity_below_the_expected_minimum_is_named_on_the_route_path() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::Unreachable, 2, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.promote_by_hand()?; @@ -13852,7 +13888,7 @@ mod tests { #[test] fn odin_itself_is_never_route_proof() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; let mut expected = routed.stub.expected.clone(); assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); // Odin provides the rendezvous capability and declares no dependency. @@ -13872,10 +13908,34 @@ mod tests { Ok(()) } + #[test] + fn a_target_providing_the_rendezvous_commits_on_odin_evidence_and_carries_the_authority() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, true)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let ready = routed.transaction()?; + assert!(ready.ready.as_ref().and_then(ReadinessEvidence::odin).is_some()); + routed.promote_by_hand()?; + routed.step()?; + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!( + generation.odin_authority, + Some(routed.world.engine.bootstrap_odin_authority.clone()) + ); + assert!(generation.latest_odin_observation.is_some()); + // Boot re-authenticates every Odin receipt this generation carries. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + Ok(()) + } + #[test] fn a_store_of_route_proof_generations_boots_and_resolves_the_bootstrap_authority() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; routed.promote_by_hand()?; From 54507060a56c370d73545a538e36fcb7a8d1eb8f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 11:02:06 +0200 Subject: [PATCH 005/264] WIP B3: pin the state rules, the lease refresh and boot re-authentication --- src/control_plane.rs | 93 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 712dc4a..49f8560 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -7192,11 +7192,6 @@ impl Engine { ) -> Result> { let mut providers = Vec::new(); for stored in &snapshot.admitted { - // A route-proof generation holds no Odin receipt to hand a - // dependent; its currency is the route observation, which is B4's. - if stored.value.ready.class() == ReadinessClass::RouteProof { - continue; - } // No pre-filter on ready == latest: a provider that published again // after going ready is still ready. rehydrate_admitted_ready owns // that judgment now, and reports why when it refuses. @@ -13908,6 +13903,94 @@ mod tests { Ok(()) } + #[test] + fn a_stateless_candidate_that_finished_warming_first_still_warms() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.stub.set_state("active"); + routed.step()?; + assert!(matches!( + routed.transaction()?.warming, + Some(WarmingEvidence::RouteProofDirect { .. }) + )); + Ok(()) + } + + #[test] + fn the_lease_refresh_asks_the_candidate_for_a_new_warming_presence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let current = resident(&routed.world)?; + let prior = routed + .world + .engine + .rehydrate_warming_token(¤t.value, now_millis()?, false)?; + let (_, evidence, token) = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?)? + .context("a warming candidate must refresh")?; + assert!(matches!(evidence, WarmingEvidence::RouteProofDirect { .. })); + assert_ne!(token.signed_presence_sha256(), prior.signed_presence_sha256()); + // A candidate that went Active is not warming: nothing to bind. + routed.stub.set_state("active"); + let failure = routed + .world + .engine + .fresh_warming_for_lease(¤t, now_millis()?) + .map(|_| ()) + .expect_err("an active candidate is not a warming one"); + assert!(format!("{failure:#}").contains("not warming"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn boot_reauthenticates_a_live_route_proof_transactions_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + + // Warming evidence that does not answer the challenge it names. + let current = resident(&routed.world)?; + let mut forged = current.value.clone(); + let Some(WarmingEvidence::RouteProofDirect { evidence }) = &mut forged.warming else { + bail!("no route-proof Warming evidence"); + }; + evidence.message_id = "candidate-another".into(); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Warming evidence for another challenge"); + assert!(format!("{failure:#}").contains("exact challenge"), "{failure:#}"); + + // A Ready label carried by a presence that was only warming. + let current = resident(&routed.world)?; + let CandidateAnswer::Answered { evidence, .. } = routed + .world + .engine + .challenge_candidate(¤t.value, &["warming"], None)? + else { + bail!("the stub candidate did not answer"); + }; + let mut forged = current.value.clone(); + forged.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + forged.ready = Some(ReadinessEvidence::RouteProof { evidence }); + replace_transaction(&routed.world.state_store, ¤t, &forged)?; + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let failure = routed + .world + .engine + .validate_durable_authority(&snapshot) + .expect_err("Ready carried by a warming presence"); + assert!(format!("{failure:#}").contains("not active"), "{failure:#}"); + routed.assert_odin_untouched()?; + Ok(()) + } + #[test] fn a_target_providing_the_rendezvous_commits_on_odin_evidence_and_carries_the_authority() -> Result<()> { From 4dcdd3608c25c0ad8d822cbc037a00d07d31d642 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 11:03:47 +0200 Subject: [PATCH 006/264] WIP B3: route admission test --- src/control_plane.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 49f8560..b1c2131 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13903,6 +13903,19 @@ mod tests { Ok(()) } + #[test] + fn route_admission_gets_past_its_gate_without_reading_odin() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + // The host's nginx is not here, so the step fails at installing the + // route: past the admission gate, which would have failed on Odin. + let failure = format!("{:#}", routed.step().expect_err("no nginx in the test world")); + assert!(failure.contains("XXXX"), "{failure}"); + routed.assert_odin_untouched()?; + Ok(()) + } + #[test] fn a_stateless_candidate_that_finished_warming_first_still_warms() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; From 5ca7760fdda8e6d910b77f00e9e4aa5ef8d17acf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 11:06:25 +0200 Subject: [PATCH 007/264] WIP B3: pin route admission past the gate --- src/control_plane.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index b1c2131..5d52938 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13905,13 +13905,21 @@ mod tests { #[test] fn route_admission_gets_past_its_gate_without_reading_odin() -> Result<()> { + // The step below would run the host's real nginx, so it only runs + // where there is none. + if Path::new("/usr/sbin/nginx").exists() { + return Ok(()); + } let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.stub.set_state("active"); routed.run_to_routing()?; - // The host's nginx is not here, so the step fails at installing the - // route: past the admission gate, which would have failed on Odin. + // The step fails at installing the route: past the admission gate, + // which would have failed on Odin. let failure = format!("{:#}", routed.step().expect_err("no nginx in the test world")); - assert!(failure.contains("XXXX"), "{failure}"); + assert!( + failure.contains("starting route actuator /usr/sbin/nginx"), + "{failure}" + ); routed.assert_odin_untouched()?; Ok(()) } From 6501cafda3e23e0db0aef58311815d420ba7ee60 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 12:46:32 +0200 Subject: [PATCH 008/264] Record B3 Soul verdict and the recipe-declares-readiness ruling Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 512ac09..0eb920b 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -11,6 +11,31 @@ on CultLib `codex/fix-node24-ajv-esm`. Behaviour cuts re-take their `file:line` anchors in their own briefs. The operator has asked for no live mitigation on Yggdrasil yet. +**B3 status, 2026-09-30 (Self).** Hands pushed B3 to `idunn/route-b3` at `5ca7760`; it is based on +B1's merge `a61540b`, not stacked on B2. The Opus Soul pass found: +- Full suite green: 252 passed, 2 ignored. +- `cargo-mutants --in-diff`: 31 caught, 4 missed. +- Soul's probe P1 drove a stateful route-proof target through the Engine, so the stateful path works; it + was only untested. + +Verdict: safe to merge for the StreamPixels web ship. The fix batch, in Hands, covers these findings: +- the evidence tag is a second class authority, and a pre-B3 transaction wedges Routing; +- the driver's rejections of a bad answer become gate waits instead of errors; +- the shortfall is prose; +- `OdinSelf` is decorative, and "is Odin" is still decided by name in five places; +- Warming is never driven past one step; +- seven unpinned checks. + +Before `idunn up`, one read-only decode of a *copy* of the live `control.cc` checks for non-terminal +transactions of routed targets with no Odin declaration (operator approved, 2026-09-30). + +**Ruling, 2026-09-30: the recipe declares readiness; Idunn infers nothing.** This settles the fork +Hands raised against a literal Q1(b). An unrouted target whose recipe declares neither a route nor an +Odin dependency is refused at admission with a typed error. It does not fall back to Odin correlation +because it has no route. `raven-muninn` and Heimdall both publish to Odin, so their recipes declare it. +The rejected alternative let Odin's availability gate Muninn's crash recovery through an undeclared, +inferred dependency. + Heads read for this map: | Repo | Ref | Notes | From 6ea824b5a9d605186150314afa50a4ce623d17d2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 13:12:46 +0200 Subject: [PATCH 009/264] B3 fix: readiness class comes from the recipe alone ReadinessClass::of is the only class authority. A target whose recipe declares neither a route nor an Odin dependency, and does not provide the rendezvous, has declared no way to prove readiness: of() returns a typed UndeclaredReadiness naming the target, and admission refuses it after plan compile, before anything is frozen or installed. The OdinCorrelated fallback for an unrouted target is gone. The evidence tag no longer decides the path. Boot re-authentication and admitted-topology refresh choose by class; a stored transaction or admitted generation whose evidence was vouched for by the other party than its Expected requires is a typed ReadinessDisagreement, reported once and held, not a step that errors every tick. Nothing re-derives around it. "Is this Odin" is the class OdinSelf, not the name "odin": validation, warming, the Odin authority lookup and the direct-warming checks read the class. canonical_sha256 on WarmingEvidence and the token's unread evidence field are deleted. A bad answer to a direct challenge is an error. The driver splits its failures into ChallengeFailure::Silent (refused, dropped, hung up, timed out) and ::Refused (a malformed, foreign, oversized or non-raw answer, or a local configuration error); only Silent waits. The capacity/disagreement shortfall is returned as a typed PresenceDisagrees, not flattened to prose. Odin-driven fixture worlds declare the Odin dependency in Expected and the correlation they sign names its provider. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 490 +++++++++++++++++++++++++++++++---------- src/deployment_plan.rs | 21 +- src/drivers.rs | 168 ++++++++++---- 3 files changed, 519 insertions(+), 160 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 5d52938..79e007f 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -37,7 +37,7 @@ use crate::deployment_plan::{ CompiledDeploymentPlan, DependencyProviderAuthority, SealedRelease, compile_deployment_plan, }; use crate::drivers::{ - CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, + ChallengeFailure, CultCacheTopologyDriver, CultCacheWriteLeaseDriver, DockerRunnerDriver, FrozenSourceReceipt, GitSourceDriver, InstalledReleaseObservation, IsolationEvidence, NginxRouteDriver, ProcessIdentity, RouteObservation, RoutePreflightReceipt, RunnerPort, SourcePort, SystemdTransientWorkloadDriver, TopologyPort, WorkloadObservation, WorkloadPort, @@ -55,7 +55,7 @@ const DEPLOYMENT_TRANSACTION_SCHEMA_V2: &str = "idunn.deployment_transaction.v2" const ADMITTED_GENERATION_SCHEMA: &str = "idunn.admitted_generation.v3"; const ADMITTED_GENERATION_SCHEMA_V2: &str = "idunn.admitted_generation.v2"; /// The capability whose declaration makes a target Odin-correlated. -const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; +pub(crate) const ODIN_RENDEZVOUS_CAPABILITY: &str = "odin.verse-rendezvous"; /// How many times continuity will restart one admitted release before it /// concludes the release itself is the problem. More than one because a start /// can fail for a passing reason -- a port still held, a peer not yet up -- @@ -276,12 +276,10 @@ impl WarmingEvidence { } } - fn canonical_sha256(&self) -> &str { + fn voucher(&self) -> Voucher { match self { - Self::OdinTopology { evidence } => &evidence.canonical_sha256, - Self::FirstOdinDirect { evidence } | Self::RouteProofDirect { evidence } => { - &evidence.canonical_sha256 - } + Self::OdinTopology { .. } | Self::FirstOdinDirect { .. } => Voucher::Odin, + Self::RouteProofDirect { .. } => Voucher::Candidate, } } @@ -303,7 +301,6 @@ impl WarmingEvidence { #[derive(Clone, Debug)] pub struct SequenceAdmittedWarming { transaction_id: String, - evidence: WarmingEvidence, signed_presence_sha256: String, runtime_instance_id: String, } @@ -311,7 +308,6 @@ pub struct SequenceAdmittedWarming { impl SequenceAdmittedWarming { fn from_topology( transaction_id: String, - evidence: TopologyEvidence, authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, ) -> Result { let record = authenticated.record(); @@ -325,7 +321,6 @@ impl SequenceAdmittedWarming { .context("Warming topology has no runtime instance")?; Ok(Self { transaction_id, - evidence: WarmingEvidence::OdinTopology { evidence }, signed_presence_sha256, runtime_instance_id, }) @@ -348,7 +343,6 @@ impl SequenceAdmittedWarming { Ok(Self { transaction_id, signed_presence_sha256: evidence.canonical_sha256.clone(), - evidence: warming, runtime_instance_id, }) } @@ -357,7 +351,6 @@ impl SequenceAdmittedWarming { pub(crate) fn for_test( transaction_id: impl Into, authenticated: AuthenticatedOdinRuntimeTopologyCorrelation, - admitted_at_unix_millis: u64, ) -> Result { let record = authenticated.record(); ensure!( @@ -370,8 +363,6 @@ impl SequenceAdmittedWarming { ); let transaction_id = transaction_id.into(); require_id(&transaction_id, "test transaction id")?; - let evidence = - TopologyEvidence::from_authenticated(&authenticated, admitted_at_unix_millis)?; let signed_presence_sha256 = record .signed_presence_sha256 .clone() @@ -380,7 +371,7 @@ impl SequenceAdmittedWarming { .runtime_instance_id .clone() .context("test Warming receipt has no runtime instance")?; - let token = Self::from_topology(transaction_id, evidence, authenticated)?; + let token = Self::from_topology(transaction_id, authenticated)?; ensure!( token.signed_presence_sha256 == signed_presence_sha256 && token.runtime_instance_id == runtime_instance_id, @@ -393,10 +384,6 @@ impl SequenceAdmittedWarming { &self.transaction_id } - pub(crate) fn evidence_sha256(&self) -> &str { - self.evidence.canonical_sha256() - } - pub(crate) fn signed_presence_sha256(&self) -> &str { &self.signed_presence_sha256 } @@ -557,6 +544,40 @@ fn route_proof_warming_states(expected: &IdunnExpectedIncarnationRecord) -> &'st } } +/// A challenged runtime answered as something that disagrees with current +/// authority. Each disagreement is named, so a capacity below the Expected +/// minimum reads as that and not as a generic refusal. It stays the error's +/// typed value; callers read `disagreements`, and the text is only its +/// rendering. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PresenceDisagrees { + pub(crate) disagreements: Vec, +} + +impl std::fmt::Display for PresenceDisagrees { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let named = self + .disagreements + .iter() + .map(|disagreement| { + format!( + "{} (expected {}, observed {})", + disagreement.code, + disagreement.expected.as_deref().unwrap_or("none"), + disagreement.observed.as_deref().unwrap_or("none"), + ) + }) + .collect::>() + .join("; "); + write!( + formatter, + "route proof answered with a runtime that disagrees with current authority: {named}" + ) + } +} + +impl std::error::Error for PresenceDisagrees {} + /// What a direct challenge to a candidate endpoint produced. enum CandidateAnswer { /// No answer: the reason it is still being waited on. @@ -567,46 +588,161 @@ enum CandidateAnswer { }, } -/// Which kind of proof admits a target as Ready. Derived from the target's -/// Expected, never stored: Expected is digest-bound, so the class cannot drift -/// from the incarnation it describes. A target that declares a -/// `shared-infrastructure odin.verse-rendezvous` dependency advertises into the -/// Verse and is Odin-correlated; a routed one that declares none is -/// route-proof. Odin itself provides the rendezvous capability rather than -/// depending on it. It collects the same receipts an Odin-correlated target -/// does, and its admitted generation is what names the Odin authority, so it -/// is never route-proof. A target with no route has no endpoint to challenge, -/// so only Odin can report on it and it stays Odin-correlated. +//// Which kind of proof admits a target as Ready, derived from what the target's +/// recipe declares and from nothing else. Idunn infers no way to prove +/// readiness: a target that provides `odin.verse-rendezvous` is Odin itself, one +/// that declares a `shared-infrastructure odin.verse-rendezvous` dependency +/// advertises into the Verse and is Odin-correlated, and a routed one that +/// declares neither answers Idunn's own challenge and is route-proof. A target +/// with no route and no Odin dependency declares no proof at all, and is +/// refused. Odin collects the same receipts an Odin-correlated target does, and +/// its admitted generation is what names the Odin authority, so it is never +/// route-proof. +/// +/// `of` is the only place a class is decided, and Expected is digest-bound, so +/// the class cannot drift from the incarnation it describes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ReadinessClass { +pub(crate) enum ReadinessClass { OdinSelf, OdinCorrelated, RouteProof, } +/// A recipe that declares neither a route nor an Odin dependency, and is not +/// Odin: it has said nothing about how it proves readiness. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct UndeclaredReadiness { + pub(crate) target: String, +} + +impl std::fmt::Display for UndeclaredReadiness { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "target {} declares no way to prove readiness: its recipe must declare a route \ + (Idunn challenges it) or a shared-infrastructure {ODIN_RENDEZVOUS_CAPABILITY} \ + dependency (Odin correlates it)", + self.target + ) + } +} + +impl std::error::Error for UndeclaredReadiness {} + impl ReadinessClass { - fn of(expected: &IdunnExpectedIncarnationRecord) -> Self { - let provides_odin = expected - .capabilities - .iter() - .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY); - let declares_odin = expected.dependencies.iter().any(|dependency| { - dependency.kind == "shared-infrastructure" - && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY - }); + /// The class of a recipe's declarations. Plan compilation and `of` both + /// call it, so admission refuses exactly what `of` would refuse. + pub(crate) fn declared( + target: &str, + provides_odin: bool, + declares_odin: bool, + routed: bool, + ) -> Result { if provides_odin { - Self::OdinSelf - } else if declares_odin || expected.route.is_none() { - Self::OdinCorrelated + Ok(Self::OdinSelf) + } else if declares_odin { + Ok(Self::OdinCorrelated) + } else if routed { + Ok(Self::RouteProof) } else { - Self::RouteProof + Err(UndeclaredReadiness { + target: target.to_owned(), + }) + } + } + + fn of(expected: &IdunnExpectedIncarnationRecord) -> Result { + Self::declared( + &expected.target, + expected + .capabilities + .iter() + .any(|capability| capability.capability == ODIN_RENDEZVOUS_CAPABILITY), + expected.dependencies.iter().any(|dependency| { + dependency.kind == "shared-infrastructure" + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + }), + expected.route.is_some(), + ) + } + + /// The class of `expected`, provided every receipt collected under it was + /// vouched for by that class's voucher. The one way a stored record is + /// asked what class it is. + fn confirmed( + expected: &IdunnExpectedIncarnationRecord, + collected: impl IntoIterator, + ) -> Result { + let required = Self::of(expected).map_err(ReadinessDisagreement::Undeclared)?; + match collected + .into_iter() + .find(|voucher| *voucher != required.voucher()) + { + None => Ok(required), + Some(collected) => Err(ReadinessDisagreement::WrongVoucher { + target: expected.target.clone(), + required, + collected, + }), + } + } + + /// Who vouches for a target of this class: Odin's correlation, or the + /// candidate answering Idunn. + fn voucher(self) -> Voucher { + match self { + Self::OdinSelf | Self::OdinCorrelated => Voucher::Odin, + Self::RouteProof => Voucher::Candidate, } } } -/// The receipt that admitted a candidate as Ready, tagged by the class of proof -/// that produced it. The tag records what was collected; the class a target -/// must collect is `ReadinessClass::of(expected)`. +/// Who vouched for a piece of stored readiness evidence. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Voucher { + Odin, + Candidate, +} + +/// A stored record whose readiness evidence is not what its own Expected +/// requires. A record admitted before the target's recipe declared how it +/// proves readiness looks like this. It is reported and held, never repaired: +/// re-deriving the evidence around it would make the evidence tag a second +/// authority for the class. +#[derive(Clone, Debug, PartialEq, Eq)] +enum ReadinessDisagreement { + Undeclared(UndeclaredReadiness), + WrongVoucher { + target: String, + required: ReadinessClass, + collected: Voucher, + }, +} + +impl std::fmt::Display for ReadinessDisagreement { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Undeclared(undeclared) => write!(formatter, "{undeclared}"), + Self::WrongVoucher { + target, + required, + collected, + } => write!( + formatter, + "target {target} is {required:?} but its stored readiness evidence was vouched \ + for by {collected:?}: it was collected under a different class, and is held \ + until the target is redeployed" + ), + } + } +} + +impl std::error::Error for ReadinessDisagreement {} + +/// The receipt that admitted a candidate as Ready, tagged by who vouched for +/// it. The tag records what was collected; what a target must collect is +/// `ReadinessClass::of(expected)`, and a tag that differs is a +/// `ReadinessDisagreement`, never a second opinion about the class. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "class", rename_all = "kebab-case", deny_unknown_fields)] enum ReadinessEvidence { @@ -615,10 +751,10 @@ enum ReadinessEvidence { } impl ReadinessEvidence { - fn class(&self) -> ReadinessClass { + fn voucher(&self) -> Voucher { match self { - Self::OdinCorrelated { .. } => ReadinessClass::OdinCorrelated, - Self::RouteProof { .. } => ReadinessClass::RouteProof, + Self::OdinCorrelated { .. } => Voucher::Odin, + Self::RouteProof { .. } => Voucher::Candidate, } } @@ -637,7 +773,7 @@ impl ReadinessEvidence { } } -/// Signed evidence that the candidate process picked up the lease Idunn +// Signed evidence that the candidate process picked up the lease Idunn /// granted: its presence carries `write_lease_sha256` equal to the grant. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -1286,6 +1422,20 @@ impl DeploymentTransaction { .is_some_and(|cleanup| !cleanup.is_complete()) } + /// Where what this transaction has collected disagrees with what its own + /// Expected requires. There is nothing to disagree with before Expected. + fn readiness_disagreement(&self) -> Option { + let expected = self.expected.as_ref()?; + ReadinessClass::confirmed( + expected, + self.warming + .iter() + .map(WarmingEvidence::voucher) + .chain(self.ready.iter().map(ReadinessEvidence::voucher)), + ) + .err() + } + fn validate(&self) -> Result<()> { ensure!( self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, @@ -1369,10 +1519,15 @@ impl DeploymentTransaction { } if let Some(ready) = &self.ready { ready.validate_shape()?; - if let (ReadinessClass::RouteProof, Some(expected)) = (ready.class(), &self.expected) { + // A candidate-vouched receipt is only ever written by this code, so + // one on a target that is not route-proof is forged. The other + // direction -- an Odin receipt on a route-proof target -- is what a + // pre-B3 record looks like, and is a ReadinessDisagreement to hold, + // not a record to refuse: refusing it would refuse to boot. + if let (Voucher::Candidate, Some(expected)) = (ready.voucher(), &self.expected) { ensure!( - ReadinessClass::of(expected) == ReadinessClass::RouteProof, - "route-proof Ready evidence for a target that declares Odin" + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" ); } } @@ -1384,14 +1539,16 @@ impl DeploymentTransaction { "Warming topology evidence exceeds the transaction replay cursor" ), WarmingEvidence::FirstOdinDirect { .. } => ensure!( - self.target == "odin", + self.expected.as_ref().is_some_and(|expected| { + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::OdinSelf) + }), "direct Warming evidence is reserved for Odin observing itself" ), WarmingEvidence::RouteProofDirect { .. } => ensure!( self.expected.as_ref().is_none_or(|expected| { - ReadinessClass::of(expected) == ReadinessClass::RouteProof + ReadinessClass::of(expected).is_ok_and(|class| class == ReadinessClass::RouteProof) }), - "direct route-proof Warming evidence for a target Odin observes" + "direct route-proof Warming evidence for a target that is not route-proof" ), } } @@ -1827,6 +1984,12 @@ struct OdinReceipts<'a> { } impl AdmittedGeneration { + /// This generation's class, or where its evidence disagrees with it. The + /// class comes from its Expected, never from the tag on its Ready receipt. + fn readiness(&self) -> Result { + ReadinessClass::confirmed(&self.expected, [self.ready.voucher()]) + } + fn odin_receipts(&self) -> Result> { match ( &self.ready, @@ -1930,8 +2093,8 @@ impl AdmittedGeneration { ); } (ReadinessEvidence::RouteProof { .. }, None, None) => ensure!( - ReadinessClass::of(&self.expected) == ReadinessClass::RouteProof, - "route-proof Ready evidence for a target that declares Odin" + ReadinessClass::of(&self.expected).is_ok_and(|class| class == ReadinessClass::RouteProof), + "route-proof Ready evidence for a target that is not route-proof" ), _ => bail!("admitted readiness evidence and Odin receipts disagree"), } @@ -2540,6 +2703,16 @@ impl ControlSnapshot { .find(|stored| stored.value.target == target) } + /// The admitted generation of the target that provides + /// `odin.verse-rendezvous`: Odin is whichever target says so, not + /// whichever is named "odin". + fn admitted_odin(&self) -> Option<&Stored> { + self.admitted.iter().find(|stored| { + ReadinessClass::of(&stored.value.expected) + .is_ok_and(|class| class == ReadinessClass::OdinSelf) + }) + } + fn transaction_for_command(&self, command_id: &str) -> Vec<&DeploymentTransaction> { let mut values = self .transactions @@ -3843,7 +4016,7 @@ impl Engine { // is, its own generation names the authority; an admitted Odin that // carries none has no authority to name, and falling back to the // bootstrap key would let a stale key vouch for the Verse. - let authority = match snapshot.admitted_for("odin") { + let authority = match snapshot.admitted_odin() { None => self.bootstrap_odin_authority.clone(), Some(stored) => stored.value.odin_authority.clone().with_context(|| { "admitted Odin generation carries no Odin authority (route-proof readiness)" @@ -3990,7 +4163,11 @@ impl Engine { let odin_authority = self.current_odin_authority(snapshot)?; for stored in &snapshot.admitted { let generation = &stored.value; - if generation.ready.class() == ReadinessClass::RouteProof { + // Only a generation whose own Expected says Odin vouches for it has + // Odin receipts to re-prove. One whose evidence disagrees with its + // class is held and reported by supervision, not re-proved here. + if !matches!(generation.readiness(), Ok(class) if class != ReadinessClass::RouteProof) + { continue; } let odin = generation.odin_receipts()?; @@ -4351,6 +4528,17 @@ impl Engine { { return Ok(false); } + // Evidence collected under another class is held and reported, not + // advanced and not repaired: no step can make it the right kind, and + // running one anyway only fails it every tick. An abort already under + // way does not read the class, so it is left to finish. + let aborting = current.value.completion.is_some() + || current.value.pre_fencing_abort.is_some() + || current.value.post_fencing_abort.is_some(); + if let Some(disagreement) = current.value.readiness_disagreement().filter(|_| !aborting) { + self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); + return Ok(false); + } let advanced = self.advance_transaction(current); if advanced.is_ok() { // The step ran and succeeded: the fault, if any, has recovered, so @@ -4582,6 +4770,13 @@ impl Engine { let mut admitted = snapshot.admitted.iter().collect::>(); admitted.sort_by_key(|stored| stored.value.target.as_str()); for current in admitted { + if let Err(disagreement) = current.value.readiness() { + self.note_fault( + "holds admitted generation", + &format!("generation:{}", current.value.target), + &anyhow::Error::new(disagreement), + ); + } let blocker = snapshot.transactions.iter().find(|stored| { stored.value.target == current.value.target && stored.value.blocks_new_target_mutation() @@ -5042,7 +5237,7 @@ impl Engine { snapshot: &ControlSnapshot, current: &Stored, ) -> Result { - if current.value.ready.class() == ReadinessClass::RouteProof { + if !matches!(current.value.readiness(), Ok(class) if class != ReadinessClass::RouteProof) { return Ok(false); } let odin = current.value.odin_receipts()?; @@ -5152,6 +5347,9 @@ impl Engine { now, &providers, )?; + // Idunn infers no way to prove readiness. A recipe that declares + // none is refused here, before anything is frozen, built or installed. + plan.readiness_class()?; return self.persist_same_phase(current, |next| { next.plan = Some(plan); next.updated_at_unix_millis = now; @@ -5333,7 +5531,8 @@ impl Engine { let snapshot = ControlSnapshot::read(&self.options.state_store)?; let incumbent_lease_sha256 = self.incumbent_lease_sha256_for_warming(&snapshot, ¤t.value)?; - if current.value.warming.is_none() && ReadinessClass::of(expected) == ReadinessClass::RouteProof { + let class = ReadinessClass::of(expected)?; + if current.value.warming.is_none() && class == ReadinessClass::RouteProof { // A route-proof target's warming is Idunn's own challenge to the // candidate endpoint. No Odin is read: the target declared no // reason to be aware of one. @@ -5375,16 +5574,17 @@ impl Engine { // Asked once. Admitting the latest topology persists the sequence // cursor, so asking twice with the same record fails the second // time as "transaction changed before topology admission". - let odin_observation = if current.value.target == "odin" - && snapshot.admitted_for("odin").is_some() + let is_odin = class == ReadinessClass::OdinSelf; + let odin_observation = if is_odin + && snapshot.admitted_odin().is_some() && current.value.command_kind != CommandKind::Continuity { self.admit_latest_topology(current, None)? } else { None }; - let odin_can_observe = current.value.target != "odin" || odin_observation.is_some(); - let odin_observes_itself = current.value.target == "odin" && !odin_can_observe; + let odin_can_observe = !is_odin || odin_observation.is_some(); + let odin_observes_itself = is_odin && !odin_can_observe; if odin_observes_itself { ensure!( expected.write_lease_required, @@ -5435,7 +5635,6 @@ impl Engine { let evidence = latest.clone(); let _token = SequenceAdmittedWarming::from_topology( admitted.value.transaction_id.clone(), - evidence.clone(), authenticated, )?; return self.persist_same_phase(&admitted, |next| { @@ -5785,7 +5984,7 @@ impl Engine { "candidate process write lease is no longer exact" ); } - if ReadinessClass::of(expected) == ReadinessClass::RouteProof { + if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { // Ready is the candidate answering Idunn's own challenge Active, // holding the exact lease Idunn granted. Once recorded it is // history; the stable route's challenge is what keeps proving it. @@ -5858,7 +6057,7 @@ impl Engine { let activation = required(¤t.value.activation, "activation")?; self.observe_candidate_before_waiting(¤t.value)?; if current.value.routing.is_none() { - let admitted = if ReadinessClass::of(expected) == ReadinessClass::RouteProof { + let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { // Ready is the candidate's own proof, recorded in AwaitingReady. // No Odin correlation is read to admit a route. ensure!( @@ -5986,7 +6185,7 @@ impl Engine { let route_proof = ReadinessClass::of(required( ¤t.value.expected, "Expected projection", - )?) == ReadinessClass::RouteProof; + )?)? == ReadinessClass::RouteProof; let ready_current = if route_proof { // The candidate's Ready proof is durable, and the stable route's // challenge below is its currency. No Odin is read to commit. @@ -6239,7 +6438,8 @@ impl Engine { // caller and failed the refresh. What the refresh owes is // freshness, and the replay check below is what provides it. ensure!( - current.value.target == "odin", + ReadinessClass::of(required(¤t.value.expected, "Warming Expected")?)? + == ReadinessClass::OdinSelf, "direct Warming refresh is reserved for Odin" ); let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; @@ -6301,7 +6501,6 @@ impl Engine { } let token = SequenceAdmittedWarming::from_topology( admitted.value.transaction_id.clone(), - fresh_evidence.clone(), authenticated, )?; if !provider_warming_advanced( @@ -6670,20 +6869,10 @@ impl Engine { )?; let correlation = correlate_runtime_presence_claim(authenticated, authority)?; if !correlation.disagreements().is_empty() { - let named = correlation - .disagreements() - .iter() - .map(|disagreement| { - format!( - "{} (expected {}, observed {})", - disagreement.code, - disagreement.expected.as_deref().unwrap_or("none"), - disagreement.observed.as_deref().unwrap_or("none"), - ) - }) - .collect::>() - .join("; "); - bail!("route proof answered with a runtime that disagrees with current authority: {named}"); + return Err(PresenceDisagrees { + disagreements: correlation.disagreements().to_vec(), + } + .into()); } let present = correlation.into_undisputed_present()?; let presence = present.record(); @@ -6727,9 +6916,10 @@ impl Engine { // Odin ever been admitted" test only contradicted that caller. What is // still checked is what this function itself depends on -- the target // is Odin, and the incarnation is stateful. + let expected = required(&transaction.expected, "Odin Expected")?; ensure!( - transaction.target == "odin" - && required(&transaction.expected, "Odin Expected")?.write_lease_required, + ReadinessClass::of(expected)? == ReadinessClass::OdinSelf + && expected.write_lease_required, "direct Warming presence is reserved for a stateful Odin incarnation" ); ensure!( @@ -7027,7 +7217,6 @@ impl Engine { ); SequenceAdmittedWarming::from_topology( transaction.transaction_id.clone(), - evidence, authenticated, ) } @@ -7097,7 +7286,9 @@ impl Engine { /// One direct challenge to a route-proof target's candidate endpoint. /// Failing to reach or hear the candidate is only silence: a process that /// is still starting does not answer, and that is waiting, not a fault. - /// What does answer is authenticated, and a bad answer is an error. + /// What does answer is authenticated, and a bad answer is an error, as is + /// a challenge Idunn could not make: those stay a `ChallengeFailure` in the + /// error's chain. fn challenge_candidate( &self, transaction: &DeploymentTransaction, @@ -7117,16 +7308,13 @@ impl Engine { let challenged_at_unix_millis = now_millis()?; let response = match driver.request_candidate_runtime_presence(expected, &message_id) { Ok(response) => response, - Err(error) => { + Err(ChallengeFailure::Silent(error)) => { return Ok(CandidateAnswer::Silent(format!( "candidate endpoint did not answer its challenge: {error:#}" ))); } + Err(bad @ ChallengeFailure::Refused(_)) => return Err(bad.into()), }; - ensure!( - response.message_id == message_id, - "candidate transport substituted its challenge identity" - ); let admitted_at_unix_millis = now_millis()?; let authority = self.runtime_authority(transaction)?; let present = self.authenticate_challenged_presence( @@ -10009,7 +10197,7 @@ mod tests { lifted.odin_publisher_sequence_cursor, legacy.odin_publisher_sequence_cursor ); - assert_eq!(lifted.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); // The repair intent still decides whatever it decided. assert_eq!( lifted.route_repair_started_at_unix_millis, @@ -10176,18 +10364,36 @@ mod tests { dependency.kind == "shared-infrastructure" && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY })); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinCorrelated); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); for dependency in &mut expected.dependencies { // Same capability, other kind: only the shared-infrastructure // declaration makes a target Odin-correlated. dependency.kind = "required".into(); } - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); expected.dependencies.clear(); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); - // Nothing to challenge: only Odin can report on an unrouted target. + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); + // Nothing to challenge and no Odin dependency: the recipe has declared + // no way to prove readiness, and Idunn does not pick one. expected.route = None; - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinCorrelated); + assert_eq!( + ReadinessClass::of(&expected), + Err(UndeclaredReadiness { + target: expected.target.clone() + }) + ); + // Declaring the Odin dependency is the declaration, routed or not: a + // routed target that only publishes to Odin is Odin-correlated. + let odin_dependency = generation.expected.dependencies.clone(); + expected.dependencies = odin_dependency.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + expected.route = generation.expected.route.clone(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinCorrelated)); + // Providing the rendezvous is being Odin, whatever else is declared. + expected.dependencies.clear(); + expected.route = None; + expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); Ok(()) } @@ -10522,6 +10728,27 @@ mod tests { }) } + /// The declaration that makes an unrouted fixture target Odin-correlated. + /// Idunn infers no way to prove readiness, and a target with no route has + /// nothing for Idunn to challenge, so the world says outright that Odin + /// reports on it. The plan selects no provider: these worlds never plan + /// against Odin, they only read its correlation, which names the provider + /// Expected does. + fn declare_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.dependencies.push(cultnet_rs::IdunnExpectedDependency { + kind: "shared-infrastructure".into(), + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + startup: "before-promotion".into(), + provider_id: Some("odin".into()), + provider_authority: Some("managed-incarnation".into()), + provider_expected_projection_sha256: Some(sha256_id(b"odin expected projection")), + provider_endpoint: None, + }); + } + /// A transaction for an unrouted, stateless target, sitting at `phase` /// (Warming or later) with every earlier phase's evidence in place. The /// plan, sealed release, Expected, activation and Warming evidence are the @@ -10607,8 +10834,9 @@ mod tests { (plan, release) } }; - let expected = release.expected_projection(&plan)?; + let mut expected = release.expected_projection(&plan)?; assert!(!expected.write_lease_required && expected.route.is_none()); + declare_odin(&mut expected); let now = now_millis()?; let command = DeploymentCommand { @@ -10755,7 +10983,25 @@ mod tests { runtime_instance_id: Some(activation.runtime_instance_id.clone()), present: true, ready, - dependencies: Vec::new(), + dependencies: expected + .dependencies + .iter() + .map(|dependency| cultnet_rs::OdinRuntimeDependencyEvidence { + kind: dependency.kind.clone(), + capability: dependency.capability.clone(), + schema: dependency.schema.clone(), + compatibility: dependency.compatibility.clone(), + provider_id: dependency.provider_id.clone(), + provider_authority: dependency.provider_authority.clone(), + provider_expected_projection_sha256: dependency + .provider_expected_projection_sha256 + .clone(), + provider_endpoint: dependency.provider_endpoint.clone(), + observed_capacity: Some(dependency.minimum_capacity), + provider_evidence_sha256: Some(sha256_id(b"odin provider evidence")), + ready: true, + }) + .collect(), disagreements: Vec::new(), signer_identity_id: world.odin_signer.entry().identity_id.clone(), publisher_sequence: sequence, @@ -11029,6 +11275,10 @@ mod tests { let (_, mut odin) = fixture_generation(FIXTURE_GENERATION)?; odin.target = "odin".into(); + // Odin is the target that provides the rendezvous, whatever it is named. + odin.expected.target = "odin".into(); + odin.expected.dependencies.clear(); + odin.expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( &world.odin_signer.trust_anchor()?, )?); @@ -11081,7 +11331,7 @@ mod tests { #[test] fn a_route_proof_generation_is_admitted_only_in_its_exact_shape() -> Result<()> { let generation = route_proof_generation()?; - assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert_eq!(generation.ready.voucher(), Voucher::Candidate); assert!(generation.odin_authority.is_none()); assert!(generation.latest_odin_observation.is_none()); assert!(generation.odin_receipts().is_err()); @@ -11247,7 +11497,7 @@ mod tests { assert_eq!(snapshot.admitted.len(), 1); let lifted = &snapshot.admitted[0].value; assert_eq!(lifted.target, envelope.key); - assert_eq!(lifted.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!(lifted.ready.voucher(), Voucher::Odin); assert!(snapshot.admitted_for(&envelope.key).is_some()); // The lifted record is held to the same key rule as a current one. @@ -13270,6 +13520,10 @@ mod tests { candidate_hits: AtomicUsize, stable_hits: AtomicUsize, stop: AtomicBool, + /// Accept the connection and close it without a word: a process + /// that is up but not answering. Unlike a closed port, no other + /// test's listener can be handed the address. + hang_up: AtomicBool, } impl RuntimeStub { @@ -13399,7 +13653,9 @@ mod tests { &stub.stable_hits }; hits.fetch_add(1, Ordering::SeqCst); - let _ = stub.serve_one(stream); + if !stub.hang_up.load(Ordering::SeqCst) { + let _ = stub.serve_one(stream); + } } Err(_) => std::thread::sleep(Duration::from_millis(5)), } @@ -13508,11 +13764,11 @@ mod tests { assert!(expected.route.is_some() && expected.dependencies.is_empty()); assert_eq!( ReadinessClass::of(&expected), - if provides_odin { + Ok(if provides_odin { ReadinessClass::OdinSelf } else { ReadinessClass::RouteProof - } + }) ); let now = now_millis()?; @@ -13572,6 +13828,7 @@ mod tests { candidate_hits: AtomicUsize::new(0), stable_hits: AtomicUsize::new(0), stop: AtomicBool::new(false), + hang_up: AtomicBool::new(false), }); stub.listen(candidate_listener, true); stub.listen(stable_listener, false); @@ -13798,7 +14055,7 @@ mod tests { let generation = admitted(&routed)?; generation.validate()?; - assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert_eq!(generation.ready.voucher(), Voucher::Candidate); assert!(generation.odin_authority.is_none()); assert!(generation.latest_odin_observation.is_none()); assert!(generation.route_supervision.is_some()); @@ -13827,7 +14084,7 @@ mod tests { routed.promote_by_hand()?; routed.step()?; let generation = admitted(&routed)?; - assert_eq!(generation.ready.class(), ReadinessClass::RouteProof); + assert_eq!(generation.ready.voucher(), Voucher::Candidate); // The valid, not-Ready correlation was neither admitted as a // receipt nor consulted: the record carries no Odin cursor. assert_eq!(generation.odin_publisher_sequence_cursor, 0); @@ -13847,10 +14104,9 @@ mod tests { assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); routed.assert_odin_untouched()?; - // Nothing listens: the same step waits, and says why. + // Nothing answers: the same step waits, and says why. let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; - silent.stub.stop.store(true, Ordering::SeqCst); - std::thread::sleep(Duration::from_millis(50)); + silent.stub.hang_up.store(true, Ordering::SeqCst); silent.step()?; let waiting = silent.transaction()?; assert!(waiting.warming.is_none() && waiting.pre_fencing_abort.is_none()); @@ -13885,11 +14141,11 @@ mod tests { fn odin_itself_is_never_route_proof() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; let mut expected = routed.stub.expected.clone(); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::RouteProof); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::RouteProof)); // Odin provides the rendezvous capability and declares no dependency. expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); assert!(expected.dependencies.is_empty()); - assert_eq!(ReadinessClass::of(&expected), ReadinessClass::OdinSelf); + assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); // Route-proof evidence for it is refused: its admitted generation // is what names the Odin authority, so it must carry one. @@ -13899,7 +14155,7 @@ mod tests { odin.ready = route_proof_evidence(); odin.latest_odin_observation = None; odin.odin_authority = None; - assert!(error_text(odin.validate()).contains("declares Odin")); + assert!(error_text(odin.validate()).contains("not route-proof")); Ok(()) } @@ -14024,7 +14280,7 @@ mod tests { routed.step()?; let generation = admitted(&routed)?; generation.validate()?; - assert_eq!(generation.ready.class(), ReadinessClass::OdinCorrelated); + assert_eq!(generation.ready.voucher(), Voucher::Odin); assert_eq!( generation.odin_authority, Some(routed.world.engine.bootstrap_odin_authority.clone()) diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index d4b1cc4..fc65c4d 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -10,7 +10,7 @@ use cultnet_rs::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use crate::control_plane::SequenceAdmittedReady; +use crate::control_plane::{ODIN_RENDEZVOUS_CAPABILITY, ReadinessClass, SequenceAdmittedReady}; use crate::deployment::{ CapabilityDependency, DeadlineBinding, DependencyKind, ExternalCapabilityBinding, OperatorBinding, ServiceTransport, SourceSelectionPolicy, StartupOrder, StateDeclaration, TargetDeclaration, @@ -640,6 +640,25 @@ impl CompiledDeploymentPlan { Ok((declaration, binding)) } + /// How this plan's target proves readiness, from its recipe's declarations + /// and its binding's route, or the typed refusal that it declares none. + /// Admission asks this before anything is sealed or installed. + pub(crate) fn readiness_class(&self) -> Result { + let (declaration, binding) = self.parsed_inputs()?; + Ok(ReadinessClass::declared( + &declaration.target, + declaration + .provides + .iter() + .any(|provided| provided.capability == ODIN_RENDEZVOUS_CAPABILITY), + declaration.dependencies.iter().any(|dependency| { + dependency.kind == DependencyKind::SharedInfrastructure + && dependency.capability == ODIN_RENDEZVOUS_CAPABILITY + }), + binding.route.is_some(), + )?) + } + fn recomputed_plan_id(&self) -> Result { let mut unsigned = self.clone(); unsigned.plan_id.clear(); diff --git a/src/drivers.rs b/src/drivers.rs index 44e4223..b1db416 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5453,7 +5453,7 @@ impl NginxRouteDriver { && route.stable_endpoint == self.binding.stable_endpoint, "route challenge binding differs from Expected" ); - self.request_runtime_presence_at(expected, message_id, &route.stable_endpoint) + Ok(self.request_runtime_presence_at(expected, message_id, &route.stable_endpoint)?) } /// The bootstrap exception for the first managed Odin observes Warming on @@ -5462,16 +5462,17 @@ impl NginxRouteDriver { &self, expected: &IdunnExpectedIncarnationRecord, message_id: &str, - ) -> Result { + ) -> Result { let route = expected .route .as_ref() - .context("candidate challenge has no Expected route")?; - ensure!( + .context("candidate challenge has no Expected route") + .map_err(refused)?; + ensure_or_refuse( route.route_id == self.binding.route_id, - "candidate challenge binding differs from Expected" - ); - self.render(expected)?; + "candidate challenge binding differs from Expected", + )?; + self.render(expected).map_err(refused)?; self.request_runtime_presence_at(expected, message_id, &route.candidate_endpoint) } @@ -5480,13 +5481,14 @@ impl NginxRouteDriver { expected: &IdunnExpectedIncarnationRecord, message_id: &str, endpoint: &str, - ) -> Result { - expected.validate()?; - require_driver_id(message_id, "route challenge message")?; + ) -> Result { + expected.validate().map_err(refused)?; + require_driver_id(message_id, "route challenge message").map_err(refused)?; let route = expected .route .as_ref() - .context("route challenge has no Expected route")?; + .context("route challenge has no Expected route") + .map_err(refused)?; let request = CultNetMessage::SnapshotRequest { message_id: message_id.to_owned(), schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), @@ -5496,27 +5498,34 @@ impl NginxRouteDriver { "http" => "http://", "tcp" => "tcp://", "rudp" => "rudp://", - _ => bail!("route challenge transport is unsupported"), + _ => return Err(refused(anyhow!("route challenge transport is unsupported"))), }; - let (host, port) = endpoint_host_port(endpoint, endpoint_prefix)?; + let (host, port) = endpoint_host_port(endpoint, endpoint_prefix).map_err(refused)?; let target = SocketAddr::new( host.parse() - .context("route challenge endpoint host is not an IP address")?, + .context("route challenge endpoint host is not an IP address") + .map_err(refused)?, port, ); let response = match route.transport.as_str() { "http" => { let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)?; let response = request_http_snapshot(target, &payload)?; - decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)? } "tcp" => { let payload = - encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0)?; + encode_cultnet_message_to_vec(&request, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)?; let response = request_tcp_snapshot(target, &payload)?; - decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0)? + decode_cultnet_message_from_slice(&response, CultNetWireContract::CultNetSchemaV0) + .map_err(refused)? } + // The RUDP client reports every failure as prose, so a bad answer + // cannot be told from no answer here: all of it reads as silence. "rudp" => request_raw_snapshot_from_rudp_catalog(CultMeshRudpSnapshotOptions { target, runtime_id: "idunn-route-observer".into(), @@ -5524,10 +5533,11 @@ impl NginxRouteDriver { schema_ids: Some(vec![GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into()]), record_keys: Some(vec![expected.target.clone()]), ..CultMeshRudpSnapshotOptions::default() - })?, - _ => bail!("route challenge transport is unsupported"), + }) + .map_err(silent)?, + _ => return Err(refused(anyhow!("route challenge transport is unsupported"))), }; - exact_route_snapshot_response(response, message_id, &expected.target) + exact_route_snapshot_response(response, message_id, &expected.target).map_err(refused) } /// Restore the exact preflight baseline. This may only replace the exact @@ -5555,51 +5565,125 @@ impl NginxRouteDriver { } } -fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { +/// Why a direct presence challenge produced no document to authenticate. +#[derive(Debug)] +pub enum ChallengeFailure { + /// Nothing answered: the connection was refused, timed out, or dropped, or + /// the peer closed it without a byte. A process still starting does this, + /// and it is waiting, not a fault. + Silent(anyhow::Error), + /// Something answered wrongly, or the challenge could not be made: a + /// malformed or foreign answer, an oversized one, or a local configuration + /// error. Waiting does not fix it. + Refused(anyhow::Error), +} + +impl std::fmt::Display for ChallengeFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Silent(error) | Self::Refused(error) => write!(formatter, "{error:#}"), + } + } +} + +impl std::error::Error for ChallengeFailure { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Silent(error) | Self::Refused(error) => Some(error.as_ref()), + } + } +} + +fn silent(error: impl Into) -> ChallengeFailure { + ChallengeFailure::Silent(error.into()) +} + +fn refused(error: impl Into) -> ChallengeFailure { + ChallengeFailure::Refused(error.into()) +} + +fn ensure_or_refuse(condition: bool, message: &'static str) -> Result<(), ChallengeFailure> { + if condition { + Ok(()) + } else { + Err(refused(anyhow!(message))) + } +} + +fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) - .with_context(|| format!("connecting stable CultNet TCP route {target}"))?; - stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; - stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; - stream.write_all(&encode_frame(payload)?)?; - stream.flush()?; + .with_context(|| format!("connecting stable CultNet TCP route {target}")) + .map_err(silent)?; + stream + .set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; + stream + .set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; + stream + .write_all(&encode_frame(payload).map_err(refused)?) + .and_then(|()| stream.flush()) + .map_err(silent)?; let mut header = [0_u8; 4]; stream .read_exact(&mut header) - .context("reading stable CultNet TCP response frame")?; + .context("reading stable CultNet TCP response frame") + .map_err(silent)?; let length = u32::from_be_bytes(header) as usize; - ensure!( + ensure_or_refuse( (1..=ROUTE_SNAPSHOT_MAX_BYTES).contains(&length), - "stable CultNet TCP response exceeds the route observation bound" - ); + "stable CultNet TCP response exceeds the route observation bound", + )?; let mut response = vec![0_u8; length]; stream .read_exact(&mut response) - .context("reading stable CultNet TCP response payload")?; + .context("reading stable CultNet TCP response payload") + .map_err(silent)?; Ok(response) } -fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result> { +fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) - .with_context(|| format!("connecting stable CultNet HTTP route {target}"))?; - stream.set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; - stream.set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT))?; + .with_context(|| format!("connecting stable CultNet HTTP route {target}")) + .map_err(silent)?; + stream + .set_read_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; + stream + .set_write_timeout(Some(ROUTE_SNAPSHOT_TIMEOUT)) + .map_err(refused)?; let head = format!( "POST {ROUTE_HTTP_SNAPSHOT_PATH} HTTP/1.1\r\nHost: {target}\r\nContent-Type: application/msgpack\r\nAccept: application/msgpack\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", payload.len() ); - stream.write_all(head.as_bytes())?; - stream.write_all(payload)?; - stream.flush()?; + stream + .write_all(head.as_bytes()) + .and_then(|()| stream.write_all(payload)) + .and_then(|()| stream.flush()) + .map_err(silent)?; let maximum = ROUTE_HTTP_MAX_HEADER_BYTES .checked_add(ROUTE_SNAPSHOT_MAX_BYTES) - .context("route HTTP response bound overflow")?; + .context("route HTTP response bound overflow") + .map_err(refused)?; let mut response = Vec::new(); (&mut stream) .take((maximum + 1) as u64) .read_to_end(&mut response) - .context("reading stable CultNet HTTP response")?; + .context("reading stable CultNet HTTP response") + .map_err(silent)?; + if response.is_empty() { + return Err(silent(anyhow!( + "stable CultNet HTTP route closed the connection without answering" + ))); + } + parse_http_snapshot(&response, maximum).map_err(refused) +} + +/// The body of an HTTP snapshot response that arrived, or why what arrived is +/// not one. +fn parse_http_snapshot(response: &[u8], maximum: usize) -> Result> { ensure!( response.len() <= maximum, "stable CultNet HTTP response exceeds the route observation bound" @@ -10194,7 +10278,7 @@ mod tests { maximum_future_skew_millis: 5, }, )?; - let warming = SequenceAdmittedWarming::for_test("test-transaction", warming, 120)?; + let warming = SequenceAdmittedWarming::for_test("test-transaction", warming)?; Ok((expected, activation, warming, provider.trust_anchor()?)) } From e11b457458924685698d93a490477bedcb4f7634 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 13:32:42 +0200 Subject: [PATCH 010/264] B3 fix: pin the route-proof rules, and delete guards another owner already holds Tests for each rule the Soul pass found unpinned, each written to fail under its own mutation: - a stateful route-proof target through the Engine, Fencing to commit: the lease is prepared and granted, Ready is refused without the exact lease and recorded with it, boot re-authenticates the store at every stage and Odin is never read; - Warming challenges once and then advances on every tick; - a stateful candidate may not answer its Warming challenge active; - a presence holds the write lease exactly when it is not warming, and is minted after its challenge; - direct Warming evidence is bound to the bytes and digest it authenticated; - candidate-vouched evidence is refused on a target Odin reports on; - a pre-B3 transaction and a pre-B3 generation over a route-proof target are held and reported once, not stepped every tick, and not re-derived; - a stored record with no declared readiness is held; - Odin is the class OdinSelf, not the name "odin"; - admission refuses an undeclared recipe with a typed error, before the source is frozen; a recipe that declares its proof is planned; - the driver's challenge failures: silence waits, every bad answer and every local configuration error is an error; - the capacity shortfall is read as a typed PresenceDisagrees. Deleted because another owner already holds them, so no test can reach them: the Routing and commit "route-proof Ready evidence" ensures (validation refuses a missing receipt, the hold refuses the other party's), the write lease comparison's warming arm (the authenticator refuses a warming presence that claims a lease), and the direct-Warming refresh check (rehydration authenticates the recorded answer as Odin's first). Engine.source is a SourcePort, the seam the admission test needs. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 1059 +++++++++++++++++++++++++++++++++++++--- src/deployment_plan.rs | 63 +++ src/drivers.rs | 273 +++++++++++ 3 files changed, 1334 insertions(+), 61 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 79e007f..8d33260 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -651,7 +651,9 @@ impl ReadinessClass { } } - fn of(expected: &IdunnExpectedIncarnationRecord) -> Result { + pub(crate) fn of( + expected: &IdunnExpectedIncarnationRecord, + ) -> Result { Self::declared( &expected.target, expected @@ -3828,7 +3830,7 @@ struct Engine { idunn_signer: ServiceIdentitySigner, idunn_anchor: ServiceIdentityTrustAnchor, bootstrap_odin_authority: AdmittedOdinAuthority, - source: GitSourceDriver, + source: Arc, docker_runner: DockerRunnerDriver, systemd_workload: Arc, host_actuators: Option, @@ -3873,11 +3875,11 @@ impl Engine { let odin_anchor = read_trust_anchor::(&options.odin_trust_anchor) .context("reading bootstrap Odin topology anchor")?; let bootstrap_odin_authority = AdmittedOdinAuthority::from_anchor(&odin_anchor)?; - let source = GitSourceDriver::new( + let source: Arc = Arc::new(GitSourceDriver::new( &options.source_root, options.staging_root.join("frozen-sources"), options.source_identity, - ); + )); let host_actuators = options .host_actuator_bind .map(|bind| HostActuatorHub::bind(bind).map(|hub| Arc::new(Mutex::new(hub)))) @@ -6059,14 +6061,9 @@ impl Engine { if current.value.routing.is_none() { let admitted = if ReadinessClass::of(expected)? == ReadinessClass::RouteProof { // Ready is the candidate's own proof, recorded in AwaitingReady. - // No Odin correlation is read to admit a route. - ensure!( - matches!( - current.value.ready, - Some(ReadinessEvidence::RouteProof { .. }) - ), - "route-proof Routing without route-proof Ready evidence" - ); + // Validation refuses Routing without a Ready receipt, and a + // receipt the other party vouched for is held before any step + // runs. No Odin correlation is read to admit a route. current.clone() } else { let current_lease = current @@ -6189,13 +6186,6 @@ impl Engine { let ready_current = if route_proof { // The candidate's Ready proof is durable, and the stable route's // challenge below is its currency. No Odin is read to commit. - ensure!( - matches!( - current.value.ready, - Some(ReadinessEvidence::RouteProof { .. }) - ), - "route-proof commit without route-proof Ready evidence" - ); current.clone() } else { let current_lease_sha256 = current @@ -6437,11 +6427,8 @@ impl Engine { // with the narrower "no admitted Odin" test contradicted the // caller and failed the refresh. What the refresh owes is // freshness, and the replay check below is what provides it. - ensure!( - ReadinessClass::of(required(¤t.value.expected, "Warming Expected")?)? - == ReadinessClass::OdinSelf, - "direct Warming refresh is reserved for Odin" - ); + // That the target is Odin was already required by the + // rehydration above, which authenticates the recorded answer. let (evidence, present) = self.observe_first_odin_warming(¤t.value)?; let warming = WarmingEvidence::FirstOdinDirect { evidence }; let token = SequenceAdmittedWarming::from_direct_presence( @@ -6890,11 +6877,11 @@ impl Engine { presence.detail == format!("route-observation:{message_id}"), "route proof response is not bound to the exact challenge" ); - let lease_is_current = if presence.state == "warming" { - presence.write_lease_sha256.is_none() - } else { - presence.write_lease_sha256.as_deref() == current_write_lease_sha256 - }; + // A warming presence is exempt from the lease comparison: it is + // Idunn's own lease that a warming candidate cannot hold yet, and the + // authenticator above already refuses one that claims a lease. + let lease_is_current = presence.state == "warming" + || presence.write_lease_sha256.as_deref() == current_write_lease_sha256; ensure!( lease_is_current, "route proof runtime does not hold the exact current process write lease" @@ -10392,7 +10379,7 @@ mod tests { // Providing the rendezvous is being Odin, whatever else is declared. expected.dependencies.clear(); expected.route = None; - expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + provide_odin(&mut expected); assert_eq!(ReadinessClass::of(&expected), Ok(ReadinessClass::OdinSelf)); Ok(()) } @@ -10728,6 +10715,16 @@ mod tests { }) } + /// What makes a target Odin: it provides the rendezvous. + fn provide_odin(expected: &mut IdunnExpectedIncarnationRecord) { + expected.capabilities.push(cultnet_rs::IdunnExpectedCapability { + capability: ODIN_RENDEZVOUS_CAPABILITY.into(), + schema: "odin.verse-topology.v1".into(), + compatibility: "v1".into(), + minimum_capacity: 1, + }); + } + /// The declaration that makes an unrouted fixture target Odin-correlated. /// Idunn infers no way to prove readiness, and a target with no route has /// nothing for Idunn to challenge, so the world says outright that Odin @@ -11278,7 +11275,7 @@ mod tests { // Odin is the target that provides the rendezvous, whatever it is named. odin.expected.target = "odin".into(); odin.expected.dependencies.clear(); - odin.expected.capabilities[0].capability = ODIN_RENDEZVOUS_CAPABILITY.into(); + provide_odin(&mut odin.expected); odin.odin_authority = Some(AdmittedOdinAuthority::from_anchor( &world.odin_signer.trust_anchor()?, )?); @@ -13494,6 +13491,142 @@ mod tests { Ok(()) } + /// A source that resolves one fixed recipe and never freezes anything. + struct FixedSource { + recipe: String, + } + + impl SourcePort for FixedSource { + fn resolve( + &self, + _binding: &OperatorBinding, + _resolution_id: &str, + _selected_at_unix_millis: u64, + ) -> Result { + Ok(crate::drivers::ResolvedSource { + facts: crate::deployment_plan::tests::source(&self.recipe), + recipe_bytes: self.recipe.clone().into_bytes(), + }) + } + + fn freeze(&self, _: &str, _: &CompiledDeploymentPlan) -> Result { + bail!("this source freezes nothing") + } + + fn observe_frozen( + &self, + _: &CompiledDeploymentPlan, + _: &FrozenSourceReceipt, + ) -> Result { + bail!("this source freezes nothing") + } + + fn cleanup(&self, _: &str, _: Option<&FrozenSourceReceipt>) -> Result<()> { + Ok(()) + } + } + + /// A Deploy transaction for an unrouted target, still Sealing, whose + /// source resolves `recipe`. One scheduler tick admits its plan or refuses it. + fn sealing_world(recipe: &str) -> Result { + use crate::deployment_plan::tests::BINDING; + let mut world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let (binding_head, binding_tail) = BINDING + .split_once("[route]") + .context("binding has no route table")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail + .split_once("[brakes]") + .context("binding has no brakes table")? + .1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + world.engine.source = Arc::new(FixedSource { + recipe: recipe.to_owned(), + }); + let now = now_millis()?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: "up-service".into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "test".into(), + requested_at_unix_millis: 100, + }; + let transaction = DeploymentTransaction::new(&command, "service".into(), 0, None, now)?; + assert!( + SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[ + CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }, + CultCacheExpectedEnvelope { + r#type: DeploymentTransaction::TYPE.into(), + key: transaction.transaction_id.clone(), + current: None, + }, + ], + &[ + command_envelope(&command, command.requested_at_unix_millis)?, + transaction_envelope(&transaction, now)?, + ], + )? + ); + Ok(world) + } + + fn unrouted_recipe() -> String { + crate::deployment_plan::tests::RECIPE + .split("[[dependencies]]") + .next() + .expect("the recipe has a body") + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ) + } + + #[test] + fn admission_refuses_a_recipe_that_declares_no_way_to_prove_readiness() -> Result<()> { + let recipe = unrouted_recipe(); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let refused = latest(&world)?; + assert!(refused.plan.is_none(), "the refused recipe was planned"); + assert!(refused.frozen_source.is_none(), "the refused recipe was frozen"); + let abort = refused + .pre_fencing_abort + .context("a recipe with no declared proof of readiness must be refused")?; + assert!( + abort.error.contains("target service declares no way to prove readiness"), + "{}", + abort.error + ); + Ok(()) + } + + #[test] + fn admission_takes_a_recipe_that_declares_its_proof() -> Result<()> { + let recipe = unrouted_recipe().replace( + "capability = \"service.runtime\"", + &format!("capability = \"{ODIN_RENDEZVOUS_CAPABILITY}\""), + ); + let world = sealing_world(&recipe)?; + world.engine.run_scheduler_tick()?; + let planned = latest(&world)?; + assert!(planned.pre_fencing_abort.is_none(), "{:?}", planned.pre_fencing_abort); + assert_eq!( + planned.plan.as_ref().map(CompiledDeploymentPlan::readiness_class).transpose()?, + Some(ReadinessClass::OdinSelf) + ); + Ok(()) + } + // --------------------------------------------------------------------- // Route-proof readiness: a routed target that declares no Odin dependency // is admitted by Idunn's own challenges. Odin is never read. @@ -13524,6 +13657,20 @@ mod tests { /// that is up but not answering. Unlike a closed port, no other /// test's listener can be handed the address. hang_up: AtomicBool, + /// The write lease its presence claims to hold: what a candidate + /// that picked up Idunn's grant reports. + lease: Mutex>, + reply: Mutex, + } + + /// How the runtime answers a well-formed challenge. + #[derive(Clone, Copy, PartialEq, Eq)] + enum Reply { + Honest, + /// An answer to some other challenge. + ForeignChallengeId, + /// An HTTP error where the snapshot should be. + HttpFailure, } impl RuntimeStub { @@ -13558,7 +13705,7 @@ mod tests { health_contract: expected.health_contract.clone(), state: (*self.state.lock().unwrap()).into(), detail: format!("route-observation:{message_id}"), - write_lease_sha256: None, + write_lease_sha256: self.lease.lock().unwrap().clone(), signer_identity_id: self.provider.entry().identity_id.clone(), publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, observed_at_unix_millis: now_millis()?, @@ -13610,8 +13757,19 @@ mod tests { else { bail!("stub received something other than a snapshot request"); }; + let reply = *self.reply.lock().unwrap(); + if reply == Reply::HttpFailure { + stream.write_all( + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + )?; + return Ok(()); + } let response = cultnet_rs::CultNetMessage::SnapshotResponseRaw { - message_id: message_id.clone(), + message_id: if reply == Reply::ForeignChallengeId { + "candidate-foreign".to_owned() + } else { + message_id.clone() + }, documents: vec![cultnet_rs::CultNetRawDocumentRecord { schema_id: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), record_key: self.expected.target.clone(), @@ -13695,6 +13853,23 @@ mod tests { minimum_capacity: u32, phase: DeploymentPhase, provides_odin: bool, + ) -> Result { + build_routed_world(odin, minimum_capacity, phase, provides_odin, false) + } + + /// The same world for a target with process-writable state: its + /// candidate is granted a write lease and Ready must be the candidate + /// reporting exactly that lease. + fn stateful_routed_world(odin: Odin, phase: DeploymentPhase) -> Result { + build_routed_world(odin, 1, phase, false, true) + } + + fn build_routed_world( + odin: Odin, + minimum_capacity: u32, + phase: DeploymentPhase, + provides_odin: bool, + stateful: bool, ) -> Result { use crate::deployment_plan::tests::{ BINDING, RECIPE, artifact_receipt, external_input_receipt, source, @@ -13730,7 +13905,30 @@ mod tests { "/etc/nginx/idunn-stream-routes/service.conf", &config.display().to_string().replace('\\', "/"), ); - let recipe = RECIPE + let (binding, recipe_source) = if stateful { + std::fs::create_dir_all(world.root.join("lease"))?; + ( + binding.replace( + "[brakes]", + &format!( + "[process_write_lease]\nrecord_path = \"{}\"\n\n[brakes]", + world.root.join("lease/process-write-lease.cc").display() + ), + ), + RECIPE + .replace( + "writer = \"none\"\nrecovery = \"rebuildable\"\nstartup = \"open-at-start\"", + "writer = \"process-bound-single-writer\"\nrecovery = \"preserve\"\nstartup = \"create-or-open-after-write-lease\"", + ) + .replace( + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + "required_environment = [\"GAMECULT_IDUNN_CANDIDATE_BIND\", \"GAMECULT_IDUNN_PROCESS_WRITE_LEASE\", \"GAMECULT_IDUNN_RUNTIME_BUNDLE\"]", + ), + ) + } else { + (binding, RECIPE.to_owned()) + }; + let recipe = recipe_source .split("[[dependencies]]") .next() .context("recipe is empty")? @@ -13762,6 +13960,7 @@ mod tests { )?; let expected = release.expected_projection(&plan)?; assert!(expected.route.is_some() && expected.dependencies.is_empty()); + assert_eq!(expected.write_lease_required, stateful); assert_eq!( ReadinessClass::of(&expected), Ok(if provides_odin { @@ -13829,6 +14028,8 @@ mod tests { stable_hits: AtomicUsize::new(0), stop: AtomicBool::new(false), hang_up: AtomicBool::new(false), + lease: Mutex::new(None), + reply: Mutex::new(Reply::Honest), }); stub.listen(candidate_listener, true); stub.listen(stable_listener, false); @@ -13859,30 +14060,20 @@ mod tests { transaction.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); } if phase >= DeploymentPhase::Fencing { - let rendered = NginxRouteDriver::new( - transaction - .plan - .as_ref() - .unwrap() - .parsed_inputs()? - .1 - .route - .context("no route binding")?, - ) - .render(transaction.expected.as_ref().unwrap())?; - transaction.route_preflight = Some(crate::drivers::RoutePreflightReceipt { - route_id: "service".into(), - candidate_runtime_instance_id: transaction - .activation - .as_ref() - .unwrap() - .runtime_instance_id - .clone(), - candidate_membership_sha256: sha256_id(&rendered), - incumbent_runtime_instance_id: None, - incumbent_membership_sha256: None, - incumbent_configuration: None, - }); + transaction.route_preflight = Some(candidate_preflight(&transaction)?); + } + if stateful { + // The lease driver reads the Expected and the observed + // activation from the topology store, as Starting left them. + let plan = transaction.plan.as_ref().unwrap(); + let anchor = world.engine.provider_anchor_for_plan(plan)?; + let topology = world.engine.topology(); + topology.publish_expected(transaction.expected.as_ref().unwrap(), &anchor)?; + topology.publish_observed_activation( + transaction.expected.as_ref().unwrap(), + transaction.activation.as_ref().unwrap(), + transaction.workload.as_ref().unwrap(), + )?; } transaction.enter_phase(phase, now); transaction.validate()?; @@ -13947,6 +14138,38 @@ mod tests { }) } + /// The preflight receipt of a candidate whose route was never + /// installed: what Warming records before Fencing, without running the + /// host's nginx. + fn candidate_preflight( + transaction: &DeploymentTransaction, + ) -> Result { + let rendered = NginxRouteDriver::new( + transaction + .plan + .as_ref() + .unwrap() + .parsed_inputs()? + .1 + .route + .context("no route binding")?, + ) + .render(transaction.expected.as_ref().unwrap())?; + Ok(crate::drivers::RoutePreflightReceipt { + route_id: "service".into(), + candidate_runtime_instance_id: transaction + .activation + .as_ref() + .unwrap() + .runtime_instance_id + .clone(), + candidate_membership_sha256: sha256_id(&rendered), + incumbent_runtime_instance_id: None, + incumbent_membership_sha256: None, + incumbent_configuration: None, + }) + } + impl RoutedWorld { fn step(&self) -> Result<()> { self.world.engine.advance_transaction(&resident(&self.world)?) @@ -14128,11 +14351,21 @@ mod tests { routed.run_to_routing()?; routed.promote_by_hand()?; routed.stub.capacity.store(1, Ordering::SeqCst); - let failure = format!("{:#}", routed.step().expect_err("a shortfall must be refused")); + let failure = routed.step().expect_err("a shortfall must be refused"); + // The shortfall is a typed value, not a sentence to search. + let disagrees = failure + .downcast_ref::() + .with_context(|| format!("the refusal is not typed: {failure:#}"))?; + let shortfall = disagrees + .disagreements + .iter() + .find(|disagreement| disagreement.code == "expected-capability-000-capacity") + .context("the capacity shortfall is not named")?; assert!( - failure.contains("expected-capability-000-capacity"), - "{failure}" + shortfall.expected.as_deref().is_some_and(|text| text.ends_with("capacity>=2")), + "{shortfall:?}" ); + assert!(shortfall.observed.is_some(), "{shortfall:?}"); assert!(admitted(&routed).is_err()); Ok(()) } @@ -14308,5 +14541,709 @@ mod tests { ); Ok(()) } + + impl RoutedWorld { + /// Boot would accept this store, and Odin was never read. + fn assert_boots(&self) -> Result<()> { + let snapshot = ControlSnapshot::read(&self.world.state_store)?; + self.world.engine.validate_durable_authority(&snapshot)?; + self.assert_odin_untouched() + } + + /// Step until `done`, checking after every step that boot would + /// still accept the store. + fn drive_until(&self, done: impl Fn(&DeploymentTransaction) -> bool) -> Result<()> { + for _ in 0..20 { + if done(&self.transaction()?) { + return Ok(()); + } + self.step()?; + self.assert_boots()?; + } + bail!("the transaction never reached the awaited state") + } + + fn answer_now(&self) -> Result<(RuntimePresenceEvidence, cultnet_rs::VerifiedRuntimePresence)> { + match self.world.engine.challenge_candidate( + &self.transaction()?, + &["warming", "active"], + None, + )? { + CandidateAnswer::Answered { evidence, present } => Ok((evidence, present)), + CandidateAnswer::Silent(reason) => bail!("the candidate was silent: {reason}"), + } + } + } + + // ----------------------------------------------------------------- + // The stateful path, end to end through the Engine. + // ----------------------------------------------------------------- + + #[test] + fn a_stateful_route_proof_target_holds_its_lease_from_fencing_to_commit() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.assert_boots()?; + + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Prepared { .. })))?; + assert_eq!(routed.transaction()?.phase, DeploymentPhase::Leasing); + routed.drive_until(|t| matches!(t.leasing, Some(LeasingEvidence::Granted { .. })))?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let lease_sha256 = routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + + // The candidate has not picked the lease up: still warming. + routed.step()?; + assert_eq!( + routed.transaction()?.last_error.as_deref(), + Some("candidate is still warming") + ); + assert!(routed.transaction()?.ready.is_none()); + // Active without the lease Idunn granted is not Ready. + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("an active candidate without the granted lease is refused"); + assert!( + format!("{refusal:#}").contains("exact current process write lease"), + "{refusal:#}" + ); + assert!(routed.transaction()?.ready.is_none()); + // Active holding exactly that lease is. + *routed.stub.lease.lock().unwrap() = Some(lease_sha256.clone()); + routed.step()?; + assert!(matches!( + routed.transaction()?.ready, + Some(ReadinessEvidence::RouteProof { .. }) + )); + routed.assert_boots()?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + routed.promote_by_hand()?; + routed.assert_boots()?; + routed.step()?; + + let generation = admitted(&routed)?; + generation.validate()?; + assert_eq!(generation.ready.voucher(), Voucher::Candidate); + assert!(generation.expected.write_lease_required); + assert_eq!(generation.leasing.lease_sha256(), Some(lease_sha256.as_str())); + assert!(generation.odin_authority.is_none()); + routed.assert_boots()?; + Ok(()) + } + + // ----------------------------------------------------------------- + // Warming is one challenge, then the phase moves. + // ----------------------------------------------------------------- + + #[test] + fn warming_challenges_once_and_then_advances_on_every_tick() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + routed.step()?; + let recorded = routed + .transaction()? + .warming + .context("the first step recorded no Warming evidence")?; + assert_eq!(routed.stub.candidate_hits.load(Ordering::SeqCst), 1); + + // The route preflight needs the host's nginx; its receipt is recorded + // here so the phase can move on. + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + next.route_preflight = Some(candidate_preflight(&next)?); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + + routed.drive_until(|t| t.phase == DeploymentPhase::Fencing)?; + assert_eq!( + routed.stub.candidate_hits.load(Ordering::SeqCst), + 1, + "a recorded Warming answer is never asked for again" + ); + assert_eq!(routed.transaction()?.warming, Some(recorded)); + Ok(()) + } + + #[test] + fn a_stateful_candidate_may_not_answer_its_warming_challenge_active() { + let mut expected = fixture_generation(FIXTURE_GENERATION).unwrap().1.expected; + expected.write_lease_required = true; + assert_eq!(route_proof_warming_states(&expected), ["warming"]); + expected.write_lease_required = false; + assert_eq!(route_proof_warming_states(&expected), ["warming", "active"]); + } + + #[test] + fn a_stateful_candidate_that_answers_warming_active_is_refused() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + routed.stub.set_state("active"); + let refusal = routed + .step() + .expect_err("a stateful candidate cannot be active before it holds a lease"); + assert!(format!("{refusal:#}").contains("not warming"), "{refusal:#}"); + assert!(routed.transaction()?.warming.is_none()); + Ok(()) + } + + // ----------------------------------------------------------------- + // What a challenged presence must prove. + // ----------------------------------------------------------------- + + #[test] + fn a_presence_holds_the_write_lease_exactly_when_it_is_not_warming() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let lease = sha256_id(b"the granted lease"); + let other = sha256_id(b"some other lease"); + let judge = |state: &'static str, + held: Option<&str>, + current: Option<&str>, + states: &[&str]| + -> Result<()> { + let challenged = now_millis()?; + routed.stub.set_state(state); + *routed.stub.lease.lock().unwrap() = held.map(str::to_owned); + let bytes = routed.stub.answer("candidate-probe")?; + routed + .world + .engine + .authenticate_challenged_presence( + &authority, + states, + current, + "candidate-probe", + challenged, + now_millis()?, + &bytes, + ) + .map(|_| ()) + }; + let either = ["warming", "active"]; + judge("warming", None, None, &either)?; + judge("active", Some(&lease), Some(&lease), &either)?; + for (state, held, current, named) in [ + // Warming holds nothing, whatever Idunn has granted. + ("warming", Some(lease.as_str()), None, "write lease"), + ("warming", Some(lease.as_str()), Some(lease.as_str()), "write lease"), + // Anything else holds exactly the current grant. + ("active", None, Some(lease.as_str()), "exact current process write lease"), + ("active", Some(lease.as_str()), None, "exact current process write lease"), + ( + "active", + Some(other.as_str()), + Some(lease.as_str()), + "exact current process write lease", + ), + ] { + let error = judge(state, held, current, &either) + .expect_err("a presence with the wrong lease is refused"); + assert!( + format!("{error:#}").contains(named), + "{state} {held:?} {current:?}: {error:#}" + ); + } + Ok(()) + } + + #[test] + fn a_presence_minted_before_its_challenge_is_refused() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let authority = routed + .world + .engine + .runtime_authority(&routed.transaction()?)?; + let bytes = routed.stub.answer("candidate-probe")?; + let minted = now_millis()?; + // Answered before the challenge that claims it: an old presence + // replayed for a new question. + let replay = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted + 1_000, + &bytes, + ) + .expect_err("a presence minted before its challenge"); + assert!( + format!("{replay:#}").contains("minted before its challenge"), + "{replay:#}" + ); + let early = routed + .world + .engine + .authenticate_challenged_presence( + &authority, + &["warming"], + None, + "candidate-probe", + minted + 1_000, + minted, + &bytes, + ) + .expect_err("a receipt before its challenge"); + assert!(format!("{early:#}").contains("predates its challenge"), "{early:#}"); + Ok(()) + } + + #[test] + fn direct_warming_evidence_is_bound_to_the_bytes_it_authenticated() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let id = routed.transaction()?.transaction_id; + let direct = |evidence: RuntimePresenceEvidence| WarmingEvidence::RouteProofDirect { evidence }; + + let (evidence, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present)?; + + // The digest is the authenticated presence's, but the bytes are not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_bytes.push(0); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("bytes that are not the authenticated presence"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // The bytes are the authenticated presence's, but the digest is not. + let (mut evidence, present) = routed.answer_now()?; + evidence.canonical_sha256 = sha256_id(b"another presence"); + let error = SequenceAdmittedWarming::from_direct_presence(id.clone(), direct(evidence), present) + .expect_err("a digest that is not the authenticated presence's"); + assert!(format!("{error:#}").contains("differs from its authenticated"), "{error:#}"); + + // Evidence about another answer altogether. + let (other, _) = routed.answer_now()?; + let (_, present) = routed.answer_now()?; + SequenceAdmittedWarming::from_direct_presence(id, direct(other), present) + .expect_err("evidence about another answer"); + Ok(()) + } + + // ----------------------------------------------------------------- + // Validation: candidate-vouched evidence belongs to route-proof targets. + // ----------------------------------------------------------------- + + #[test] + fn candidate_vouched_evidence_is_refused_on_a_target_odin_reports_on() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let odin_reported = transaction_at(&world, DeploymentPhase::Warming)?; + assert_eq!( + ReadinessClass::of(odin_reported.expected.as_ref().unwrap()), + Ok(ReadinessClass::OdinCorrelated) + ); + + let mut warming = odin_reported.clone(); + warming.warming = Some(WarmingEvidence::RouteProofDirect { + evidence: evidence.clone(), + }); + assert!(error_text(warming.validate()).contains("not route-proof")); + + let mut ready = odin_reported.clone(); + ready.ready = Some(ReadinessEvidence::RouteProof { + evidence: evidence.clone(), + }); + assert!(error_text(ready.validate()).contains("not route-proof")); + + // Direct Odin warming is Odin's alone, by class: a route-proof + // target may not claim it, and a target that provides the + // rendezvous may, whatever it is named. + let mut direct = routed.transaction()?; + direct.warming = Some(WarmingEvidence::FirstOdinDirect { + evidence: evidence.clone(), + }); + assert!(error_text(direct.validate()).contains("reserved for Odin")); + let odin = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + let mut direct = odin.transaction()?; + assert_eq!(direct.target, "service"); + direct.warming = Some(WarmingEvidence::FirstOdinDirect { evidence }); + direct.validate()?; + Ok(()) + } + + #[test] + fn a_route_proof_transaction_past_awaiting_ready_needs_the_candidates_ready_proof() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let routing = routed.transaction()?; + routing.validate()?; + let mut bare = routing.clone(); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + routed.promote_by_hand()?; + let mut bare = routed.transaction()?; + assert_eq!(bare.phase, DeploymentPhase::Committing); + bare.ready = None; + assert!(error_text(bare.validate()).contains("lacks Ready evidence")); + Ok(()) + } + + // ----------------------------------------------------------------- + // The class is the Expected's alone. Evidence collected under another + // class is held and reported, never stepped, never repaired. + // ----------------------------------------------------------------- + + /// Odin's Ready for the world's target, as pre-B3 code recorded it for + /// every routed target. + fn odin_receipt_for( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ) -> Result { + let now = now_millis()?; + let bytes = signed_correlation(&routed.world, transaction, sequence, true)?; + let authenticated = routed.world.engine.authenticate_topology_bytes( + &ControlSnapshot::read(&routed.world.state_store)?, + transaction, + &bytes, + transaction + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256), + now, + )?; + TopologyEvidence::from_authenticated(&authenticated, now) + } + + fn stored(generation: &AdmittedGeneration) -> Stored { + Stored { + value: generation.clone(), + envelope: CultCacheEnvelope { + key: generation.target.clone(), + r#type: AdmittedGeneration::TYPE.into(), + payload: Vec::new(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + schema_id: Some(ADMITTED_GENERATION_SCHEMA.into()), + }, + } + } + + #[test] + fn a_pre_b3_transaction_over_a_route_proof_target_is_held_and_reported_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + + // Ready as pre-B3 Idunn wrote it: an Odin receipt. + let current = resident(&routed.world)?; + let mut legacy = current.value.clone(); + let receipt = odin_receipt_for(&routed, &legacy, 9)?; + legacy.latest_odin_observation = Some(receipt.clone()); + legacy.odin_publisher_sequence_cursor = 9; + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { evidence: receipt }); + replace_transaction(&routed.world.state_store, ¤t, &legacy)?; + + let disagreement = ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }; + assert_eq!(legacy.readiness_disagreement(), Some(disagreement.clone())); + // The record is readable and boot accepts it: held, not refused. + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + routed.world.engine.validate_durable_authority(&snapshot)?; + // Stepping it would fail, on every tick, for good. + assert!(routed.step().is_err()); + + let hits = ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst), + ); + for _ in 0..4 { + assert!(!routed.world.engine.run_scheduler_tick()?); + } + let held = resident(&routed.world)?; + assert_eq!(held.value.phase, DeploymentPhase::Routing); + assert!(held.value.pre_fencing_abort.is_none()); + assert!(held.value.post_fencing_abort.is_none()); + assert_eq!(held.value.ready, legacy.ready, "nothing re-derived the evidence"); + assert_eq!( + held.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()) + ); + // Reported once: later ticks leave the record alone. + routed.world.engine.run_scheduler_tick()?; + assert_eq!(resident(&routed.world)?.envelope, held.envelope); + assert_eq!( + hits, + ( + routed.stub.candidate_hits.load(Ordering::SeqCst), + routed.stub.stable_hits.load(Ordering::SeqCst) + ), + "a held transaction is asked nothing" + ); + routed.assert_odin_untouched()?; + + // An abort already under way is not held: it never reads the class. + let mut aborting = held.value.clone(); + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + replace_transaction(&routed.world.state_store, &held, &aborting)?; + let after = resident(&routed.world)?; + let _ = routed.world.engine.resume_candidate(&after); + assert_ne!( + resident(&routed.world)?.value.last_error.as_deref(), + Some(disagreement.to_string().as_str()), + "an abort in progress runs instead of being held" + ); + Ok(()) + } + + #[test] + fn a_pre_b3_generation_over_a_route_proof_target_boots_and_is_reported_not_refreshed() + -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + routed.promote_by_hand()?; + routed.step()?; + + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("nothing was admitted")?; + let receipt = odin_receipt_for(&routed, &at_routing, 9)?; + let mut legacy = current.value.clone(); + legacy.ready = ReadinessEvidence::OdinCorrelated { + evidence: receipt.clone(), + }; + legacy.latest_odin_observation = Some(receipt); + // Receipts that would not authenticate if they were re-proved: the + // authority is another key's. Held generations are not re-proved. + let stranger = cultnet_rs::enroll_service_identity_at::( + &routed.world.root.join("identities/stranger.cc"), + )?; + legacy.odin_authority = Some(AdmittedOdinAuthority::from_anchor(&stranger.trust_anchor()?)?); + legacy.odin_publisher_sequence_cursor = 9; + assert!( + SingleFileMessagePackBackingStore::new(&routed.world.state_store) + .compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: AdmittedGeneration::TYPE.into(), + key: legacy.target.clone(), + current: Some(current.envelope.clone()), + }], + &[admitted_envelope(&legacy, now_millis()?)?], + )? + ); + + assert_eq!( + legacy.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::RouteProof, + collected: Voucher::Odin, + }) + ); + let snapshot = ControlSnapshot::read(&routed.world.state_store)?; + let current = snapshot.admitted.first().context("the generation is gone")?; + // The class chooses the path: no Odin receipt is re-proved at boot + // and none is refreshed, though the evidence is an Odin receipt. + routed.world.engine.validate_durable_authority(&snapshot)?; + assert!(!routed.world.engine.refresh_admitted_topology(&snapshot, current)?); + routed.assert_odin_untouched()?; + + routed.world.engine.supervise_one_admitted_generation()?; + assert!( + routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .contains_key("generation:service"), + "supervision reports the disagreement" + ); + routed.assert_odin_untouched()?; + Ok(()) + } + + #[test] + fn a_stored_record_that_declares_no_way_to_prove_readiness_is_held() -> Result<()> { + let world = EngineFixture::new()?; + // Odin's store is unreadable: any read of it is an error. + std::fs::write(&world.engine.options.odin_correlation_store, b"odin is not a cache")?; + + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + let target = generation.expected.target.clone(); + assert_eq!(generation.readiness(), Ok(ReadinessClass::OdinCorrelated)); + generation.expected.dependencies.clear(); + generation.expected.route = None; + let undeclared = UndeclaredReadiness { target: target.clone() }; + assert_eq!( + generation.readiness(), + Err(ReadinessDisagreement::Undeclared(undeclared.clone())) + ); + assert!( + !world + .engine + .refresh_admitted_topology(&ControlSnapshot::default(), &stored(&generation))? + ); + + // The mirror: Odin declared, but the receipt is the candidate's. + let (_, mut odin_class) = fixture_generation(FIXTURE_GENERATION)?; + odin_class.ready = route_proof_evidence(); + assert_eq!( + odin_class.readiness(), + Err(ReadinessDisagreement::WrongVoucher { + target: target.clone(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + + // A transaction says the same about what it has collected. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let (evidence, _) = routed.answer_now()?; + let odin_world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&odin_world, DeploymentPhase::Warming)?; + assert_eq!(seeded.readiness_disagreement(), None); + let mut collected = seeded.clone(); + collected.warming = Some(WarmingEvidence::RouteProofDirect { evidence }); + assert_eq!( + collected.readiness_disagreement(), + Some(ReadinessDisagreement::WrongVoucher { + target: "service".into(), + required: ReadinessClass::OdinCorrelated, + collected: Voucher::Candidate, + }) + ); + let mut bare = seeded.clone(); + bare.expected.as_mut().unwrap().dependencies.clear(); + assert_eq!( + bare.readiness_disagreement(), + Some(ReadinessDisagreement::Undeclared(UndeclaredReadiness { + target: "service".into() + })) + ); + let mut no_expected = seeded; + no_expected.expected = None; + assert_eq!(no_expected.readiness_disagreement(), None); + Ok(()) + } + + // ----------------------------------------------------------------- + // Odin is whichever target provides the rendezvous. + // ----------------------------------------------------------------- + + #[test] + fn odin_is_the_target_that_provides_the_rendezvous_not_the_target_named_odin() -> Result<()> { + let world = EngineFixture::new()?; + let (_, fixture) = fixture_generation(FIXTURE_GENERATION)?; + + // A target named "odin" that merely depends on Odin is not Odin. + let mut named = fixture.clone(); + named.target = "odin".into(); + named.expected.target = "odin".into(); + let mut snapshot = ControlSnapshot::default(); + snapshot.admitted.push(stored(&named)); + assert!(snapshot.admitted_odin().is_none()); + assert_eq!( + world.engine.current_odin_authority(&snapshot)?, + world.engine.bootstrap_odin_authority + ); + + // The target that provides the rendezvous is, whatever it is called. + let mut provider = fixture.clone(); + provider.target = "verse".into(); + provider.expected.target = "verse".into(); + provider.expected.dependencies.clear(); + provide_odin(&mut provider.expected); + let authority = AdmittedOdinAuthority::from_anchor(&world.odin_signer.trust_anchor()?)?; + provider.odin_authority = Some(authority.clone()); + snapshot.admitted.push(stored(&provider)); + assert_eq!( + snapshot.admitted_odin().map(|found| found.value.target.as_str()), + Some("verse") + ); + assert_eq!( + snapshot.admitted_odin().and_then(|found| found.value.odin_authority.clone()), + Some(authority) + ); + Ok(()) + } + + #[test] + fn a_target_that_provides_the_rendezvous_warms_as_odin_whatever_it_is_named() -> Result<()> { + // Nothing else has been admitted that could observe it, so Odin + // observes itself directly -- and only a stateful incarnation may. + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true)?; + assert_eq!(routed.transaction()?.target, "service"); + let refusal = routed + .step() + .expect_err("a stateless Odin cannot be observed directly"); + assert!( + format!("{refusal:#}").contains("direct Odin warming must be a stateful incarnation"), + "{refusal:#}" + ); + Ok(()) + } + + #[test] + fn direct_odin_warming_is_reserved_to_odin_by_class_not_by_name() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + let mut named = routed.transaction()?; + named.target = "odin".into(); + let error = routed + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a target named odin that is not Odin"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + + // ----------------------------------------------------------------- + // A bad answer is an error. Only silence waits. + // ----------------------------------------------------------------- + + #[test] + fn a_bad_answer_to_a_challenge_is_an_error_and_only_silence_waits() -> Result<()> { + for reply in [Reply::ForeignChallengeId, Reply::HttpFailure] { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + *routed.stub.reply.lock().unwrap() = reply; + let error = match routed.world.engine.challenge_candidate( + &routed.transaction()?, + &["warming"], + None, + ) { + Err(error) => error, + Ok(_) => bail!("a bad answer was taken for silence"), + }; + assert!( + matches!( + error.downcast_ref::(), + Some(ChallengeFailure::Refused(_)) + ), + "{error:#}" + ); + // Before the fence a candidate that answers wrongly is aborted, + // not waited on. + routed.world.engine.run_scheduler_tick()?; + let after = routed.transaction()?; + assert!(after.pre_fencing_abort.is_some(), "the bad answer aborted it"); + assert!(after.warming.is_none()); + } + + let silent = routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, false)?; + silent.stub.hang_up.store(true, Ordering::SeqCst); + assert!(matches!( + silent.world.engine.challenge_candidate(&silent.transaction()?, &["warming"], None)?, + CandidateAnswer::Silent(_) + )); + Ok(()) + } } } diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index fc65c4d..7169601 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -1566,6 +1566,69 @@ nodes = ["yggdrasil"] ); } + #[test] + fn a_plan_names_how_its_target_proves_readiness_and_agrees_with_its_expected() { + use crate::control_plane::UndeclaredReadiness; + let unrouted_binding = { + let (head, tail) = BINDING.split_once("[route]").unwrap(); + format!("{head}[brakes]{}", tail.split_once("[brakes]").unwrap().1) + }; + let unrouted = |recipe: &str| { + recipe + .replace("route_required = true", "route_required = false") + .replace( + r#"["GAMECULT_IDUNN_CANDIDATE_BIND", "GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + r#"["GAMECULT_IDUNN_RUNTIME_BUNDLE"]"#, + ) + }; + let without_odin = RECIPE.split("[[dependencies]]").next().unwrap().to_owned(); + let provides_odin = |recipe: &str| { + recipe.replace("capability = \"service.runtime\"", "capability = \"odin.verse-rendezvous\"") + }; + let cases: [(&str, String, &str, Option, Result); 5] = [ + ("depends on Odin, routed", RECIPE.to_owned(), BINDING, Some(18001), Ok(ReadinessClass::OdinCorrelated)), + ("routed, no Odin", without_odin.clone(), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)), + ("unrouted, no Odin", unrouted(&without_odin), &unrouted_binding, None, Err(())), + ("provides Odin, unrouted", provides_odin(&unrouted(&without_odin)), &unrouted_binding, None, Ok(ReadinessClass::OdinSelf)), + ("depends on Odin, unrouted", unrouted(RECIPE), &unrouted_binding, None, Ok(ReadinessClass::OdinCorrelated)), + ]; + for (name, recipe, binding, port, outcome) in cases { + let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; + let plan = compile_deployment_plan( + recipe.as_bytes(), + binding.as_bytes(), + source(&recipe), + "service-incarnation-1", + port, + 110, + if recipe.contains("[[dependencies]]") { &providers } else { &[] }, + ) + .unwrap_or_else(|error| panic!("{name}: {error:#}")); + let release = SealedRelease::new( + &plan, + vec![artifact_receipt()], + vec![external_input_receipt()], + 120, + ) + .unwrap(); + let expected = release.expected_projection(&plan).unwrap(); + match outcome { + Ok(class) => { + assert_eq!(plan.readiness_class().unwrap(), class, "{name}"); + assert_eq!(ReadinessClass::of(&expected), Ok(class), "{name}"); + } + Err(()) => { + let error = plan.readiness_class().expect_err(name); + let undeclared = UndeclaredReadiness { + target: "service".into(), + }; + assert_eq!(error.downcast_ref::(), Some(&undeclared), "{name}"); + assert_eq!(ReadinessClass::of(&expected), Err(undeclared), "{name}"); + } + } + } + } + #[test] fn expected_projection_is_sanitized_and_names_the_selected_graph() { let plan = plan(); diff --git a/src/drivers.rs b/src/drivers.rs index b1db416..749ac22 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8118,6 +8118,279 @@ fn apply_identity(command: &mut Command, identity: Option) -> R mod tests { use super::*; + // --------------------------------------------------------------------- + // A direct challenge fails one of two ways. Silence waits; a bad answer, + // or a challenge that cannot be made, is an error. + // --------------------------------------------------------------------- + + /// A peer that reads one snapshot request and writes back whatever `reply` + /// builds from the challenge id. Returns the port it listens on. + fn scripted_peer(reply: impl FnOnce(&str) -> Vec + Send + 'static) -> u16 { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + let port = listener.local_addr().expect("a bound port").port(); + thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut request = Vec::new(); + let mut chunk = [0_u8; 4096]; + let message_id = loop { + let Ok(read) = stream.read(&mut chunk) else { + return; + }; + if read == 0 { + return; + } + request.extend_from_slice(&chunk[..read]); + let Some(end) = request.windows(4).position(|window| window == b"\r\n\r\n") else { + continue; + }; + if let Ok(CultNetMessage::SnapshotRequest { message_id, .. }) = + decode_cultnet_message_from_slice( + &request[end + 4..], + CultNetWireContract::CultNetSchemaV0, + ) + { + break message_id; + } + }; + let _ = stream.write_all(&reply(&message_id)); + }); + port + } + + fn http_ok(body: &[u8]) -> Vec { + let mut response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + response.extend_from_slice(body); + response + } + + fn raw_snapshot(message_id: &str, schema: &str, key: &str, payload: Vec) -> Vec { + http_ok( + &encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotResponseRaw { + message_id: message_id.into(), + documents: vec![cultnet_rs::CultNetRawDocumentRecord { + schema_id: schema.into(), + record_key: key.into(), + stored_at: "1970-01-01T00:00:00.100Z".into(), + payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, + payload, + source_runtime_id: None, + source_agent_id: None, + source_role: None, + tags: None, + }], + }, + CultNetWireContract::CultNetSchemaV0, + ) + .expect("an encodable response"), + ) + } + + fn honest(message_id: &str) -> Vec { + raw_snapshot( + message_id, + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "service", + vec![1, 2, 3], + ) + } + + /// One candidate challenge to `port`, with the Expected adjusted by `adjust`. + fn challenge( + port: u16, + adjust: impl FnOnce(&mut IdunnExpectedIncarnationRecord), + ) -> Result { + let temp = tempfile::tempdir().expect("a temporary directory"); + let mut candidate = expected(); + candidate.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "service".into(), + transport: "http".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + candidate_endpoint: format!("http://127.0.0.1:{port}"), + }); + adjust(&mut candidate); + NginxRouteDriver::new(RouteBinding { + driver: RouteDriver::NginxStreamTcp, + route_id: "service".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + private_host: "127.0.0.1".into(), + private_port_start: port, + private_port_end: port, + config_path: temp.path().join("service.conf"), + reload_unit: "nginx.service".into(), + }) + .request_candidate_runtime_presence(&candidate, "candidate-probe") + } + + fn assert_silent(outcome: Result, why: &str) { + assert!( + matches!(outcome, Err(ChallengeFailure::Silent(_))), + "{why} must read as silence: {outcome:?}" + ); + } + + fn assert_refused(outcome: Result, why: &str) { + assert!( + matches!(outcome, Err(ChallengeFailure::Refused(_))), + "{why} must be an error, not a wait: {outcome:?}" + ); + } + + #[test] + fn an_exact_answer_is_taken() { + let port = scripted_peer(honest); + let answer = challenge(port, |_| {}).expect("the exact answer"); + assert_eq!(answer.message_id, "candidate-probe"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + } + + #[test] + fn a_challenge_that_nothing_answers_is_silence() { + assert_silent(challenge(scripted_peer(|_| Vec::new()), |_| {}), "a peer that hangs up"); + // Half a response, then the peer is gone. + assert_refused( + challenge(scripted_peer(|_| b"HTTP/1.1 200 OK\r\nContent-Le".to_vec()), |_| {}), + "a response that never finished its headers", + ); + // Nothing listens. + let reservation = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + let port = reservation.local_addr().expect("a bound port").port(); + drop(reservation); + assert_silent(challenge(port, |_| {}), "a closed port"); + } + + #[test] + fn a_bad_answer_is_refused_not_waited_on() { + assert_refused( + challenge( + scripted_peer(|_| { + raw_snapshot( + "candidate-another", + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "service", + vec![1], + ) + }), + |_| {}, + ), + "an answer to another challenge", + ); + assert_refused( + challenge( + scripted_peer(|id| raw_snapshot(id, "some.other.schema", "service", vec![1])), + |_| {}, + ), + "a substituted document schema", + ); + assert_refused( + challenge( + scripted_peer(|id| { + raw_snapshot( + id, + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "another-target", + vec![1], + ) + }), + |_| {}, + ), + "a substituted record key", + ); + assert_refused( + challenge( + scripted_peer(|id| { + raw_snapshot( + id, + cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA, + "service", + vec![0; ROUTE_SNAPSHOT_MAX_BYTES + 1], + ) + }), + |_| {}, + ), + "an oversized payload", + ); + assert_refused( + challenge( + scripted_peer(|id| { + http_ok( + &encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: id.into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + ) + .expect("an encodable message"), + ) + }), + |_| {}, + ), + "a response that is not a raw snapshot", + ); + assert_refused( + challenge( + scripted_peer(|_| { + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_vec() + }), + |_| {}, + ), + "an HTTP error", + ); + assert_refused( + challenge(scripted_peer(|_| http_ok(b"not a cultnet message")), |_| {}), + "a body that is not CultNet", + ); + assert_refused( + challenge(scripted_peer(|_| b"HTTP/1.1 200 OK +Content-Le".to_vec()), |_| {}), + "a response that never finished its headers", + ); + } + + #[test] + fn a_challenge_that_cannot_be_made_is_an_error_not_a_wait() { + let port = scripted_peer(honest); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().route_id = "another-route".into(); + }), + "a binding that differs from Expected", + ); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().transport = "smtp".into(); + }), + "an unsupported transport", + ); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().candidate_endpoint = + format!("http://candidate.example:{port}"); + }), + "a host that is not an IP address", + ); + assert_refused( + challenge(port, |candidate| { + candidate.route.as_mut().unwrap().candidate_endpoint = + format!("http://127.0.0.1:{}", port.wrapping_add(1)); + }), + "a candidate endpoint outside the route binding", + ); + assert_refused( + challenge(port, |candidate| candidate.route = None), + "an Expected with no route", + ); + } + #[test] fn route_connect_retries_connection_refused_until_listener_is_ready() -> Result<()> { let reservation = std::net::TcpListener::bind("127.0.0.1:0")?; From 771138a164d6f2a137f315a85752a0bb3dec970d Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 13:41:17 +0200 Subject: [PATCH 011/264] Record B2 Soul rejection and the rollback / first-deploy rulings Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 0eb920b..794bc30 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -36,6 +36,30 @@ because it has no route. `raven-muninn` and Heimdall both publish to Odin, so th The rejected alternative let Odin's availability gate Muninn's crash recovery through an undeclared, inferred dependency. +**B2 status, 2026-09-30 (Self).** Hands pushed `idunn/route-b2` at `1786ddf`. The Opus Soul pass rejected +it for merge. +- **F1 (critical):** `ContinuityBackoff.deferral_reason` was added without a schema bump. B2 fails the + canonical re-encode of every three-field v3 generation that F0, B1 or B3 wrote, so Idunn cannot read its + own store. +- **F2:** the ceiling refuses a rollback, which leaves an unproven candidate on the stable route. +- **F3:** a failed challenge writes, and continuity's CAS then loses against a stale envelope. +- **F4:** neither window rolls; both are anchored at the first charge. +- **F5:** a clock stepped backwards stalls everything for the length of the step. +- **F6:** `record_proved_challenge` is unpinned. +- **F7:** the mount preflight is not gated. +- B2 also does not compile on top of B3, and after a one-line fix two tests fail, because the fixture target + is now route-proof. + +The rework runs after B3 merges and is rebased onto it. + +**Rulings, 2026-09-30.** +- **The ceiling never refuses a rollback.** Restoring the admitted or incumbent route is survival, not + deployment. It is always allowed and always counted. Only installs and other forward changes can be + refused. A rollback that keeps failing is spaced by backoff, never refused. +- **First routed deployments are metered now.** The actuation window lives on the target, not on the + admitted generation, so a target with no admitted generation is charged too. That schema change is folded + into the store-version bump that F1 needs: a v4 generation, with a typed lift from v3. + Heads read for this map: | Repo | Ref | Notes | From 8d1dc9a87e8a0b54f2862b306381e1a6774125ad Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 14:09:04 +0200 Subject: [PATCH 012/264] Record the B3 fix batch and the hold / rollout rulings Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 794bc30..1406316 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -29,6 +29,20 @@ Verdict: safe to merge for the StreamPixels web ship. The fix batch, in Hands, c Before `idunn up`, one read-only decode of a *copy* of the live `control.cc` checks for non-terminal transactions of routed targets with no Odin declaration (operator approved, 2026-09-30). +**B3 fix batch, 2026-09-30.** Pushed `6ea824b` and `e11b457`; 275 tests pass. +- `ReadinessClass::of` is the only class authority. The evidence tag is demoted to `Voucher`. +- `ChallengeFailure::{Silent, Refused}` separates silence from a refused answer. +- `PresenceDisagrees` makes the capacity shortfall a typed value. +- The recipes are on `route/b3-declare`: Heimdall `d5acc94`, Muninn `2260853`. +- Still open, for a fresh Hands: three survivors (boot re-proof by tag, config error as silence, first-Odin + check by name) and `cargo mutants --in-diff`. + +**Rulings, 2026-09-30:** +- A stored record whose evidence disagrees with its declared class is **held and reported, never aborted** + by Idunn. B5's deadlines or the operator resolve it. +- **Heimdall and raven-muninn are redeployed with their declaring recipes in the same window as Idunn B3 + and the StreamPixels ship.** + **Ruling, 2026-09-30: the recipe declares readiness; Idunn infers nothing.** This settles the fork Hands raised against a literal Q1(b). An unrouted target whose recipe declares neither a route nor an Odin dependency is refused at admission with a typed error. It does not fall back to Odin correlation From aa1889b2884697cc3c8833e8b834b5fddbea3888 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 14:41:25 +0200 Subject: [PATCH 013/264] B3 fix: kill the three surviving mutants Boot re-proof by tag: the held-generation test used receipts that still re-proved cleanly (boot reads the current Odin authority, not the stored one), so re-proving them passed. The receipts are now authentic but not Ready, so only skipping them lets the store boot. Config error as silence: the candidate path renders Expected first, which parses the same endpoints, so the challenge's own parse is reached only from the stable path. A new test drives that parse directly. First-Odin gate by name: the stub can now answer a first-Odin Warming challenge, and a stateful provider not named odin is accepted while a stateful target named odin that provides nothing is refused. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 66 ++++++++++++++++++++++++++++++++++++++++---- src/drivers.rs | 40 ++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 8d33260..84dd4d2 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13671,6 +13671,9 @@ mod tests { ForeignChallengeId, /// An HTTP error where the snapshot should be. HttpFailure, + /// A first Odin candidate answering its Warming challenge: the + /// detail names the challenge as `idunn-warming:`. + FirstOdinWarming, } impl RuntimeStub { @@ -13704,7 +13707,10 @@ mod tests { .collect(), health_contract: expected.health_contract.clone(), state: (*self.state.lock().unwrap()).into(), - detail: format!("route-observation:{message_id}"), + detail: match *self.reply.lock().unwrap() { + Reply::FirstOdinWarming => format!("idunn-warming:{message_id}"), + _ => format!("route-observation:{message_id}"), + }, write_lease_sha256: self.lease.lock().unwrap().clone(), signer_identity_id: self.provider.entry().identity_id.clone(), publisher_sequence: self.sequence.fetch_add(1, Ordering::SeqCst) + 1, @@ -14900,9 +14906,18 @@ mod tests { routed: &RoutedWorld, transaction: &DeploymentTransaction, sequence: u64, + ) -> Result { + odin_receipt_with(routed, transaction, sequence, true) + } + + fn odin_receipt_with( + routed: &RoutedWorld, + transaction: &DeploymentTransaction, + sequence: u64, + ready: bool, ) -> Result { let now = now_millis()?; - let bytes = signed_correlation(&routed.world, transaction, sequence, true)?; + let bytes = signed_correlation(&routed.world, transaction, sequence, ready)?; let authenticated = routed.world.engine.authenticate_topology_bytes( &ControlSnapshot::read(&routed.world.state_store)?, transaction, @@ -15011,14 +15026,18 @@ mod tests { let snapshot = ControlSnapshot::read(&routed.world.state_store)?; let current = snapshot.admitted.first().context("nothing was admitted")?; - let receipt = odin_receipt_for(&routed, &at_routing, 9)?; + // Receipts that would not survive being re-proved: authentic, but + // not Ready. Boot re-proves against the current Odin authority and + // requires exact semantic Ready, never what a generation stores, so + // this record is only bootable if boot does not re-prove it. + let receipt = odin_receipt_with(&routed, &at_routing, 9, false)?; let mut legacy = current.value.clone(); legacy.ready = ReadinessEvidence::OdinCorrelated { evidence: receipt.clone(), }; legacy.latest_odin_observation = Some(receipt); - // Receipts that would not authenticate if they were re-proved: the - // authority is another key's. Held generations are not re-proved. + // The stored authority is present because an Odin-tagged record + // carries one; it is not what boot reads. let stranger = cultnet_rs::enroll_service_identity_at::( &routed.world.root.join("identities/stranger.cc"), )?; @@ -15205,6 +15224,43 @@ mod tests { Ok(()) } + #[test] + fn a_stateful_target_that_provides_the_rendezvous_is_observed_directly_whatever_it_is_named() + -> Result<()> { + let provider = build_routed_world(Odin::Unreachable, 1, DeploymentPhase::Warming, true, true)?; + let transaction = provider.transaction()?; + assert_eq!(transaction.target, "service"); + assert!(transaction.expected.as_ref().unwrap().write_lease_required); + provider.stub.set_state("warming"); + *provider.stub.reply.lock().unwrap() = Reply::FirstOdinWarming; + let challenged_at = now_millis()?; + let answer = provider.stub.answer("warming-probe")?; + provider.world.engine.authenticate_first_odin_warming_presence( + &transaction, + "warming-probe", + challenged_at, + now_millis()?, + &answer, + )?; + + // The reverse: named odin, stateful, but nothing provides the + // rendezvous. + let stateful = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Warming)?; + let mut named = stateful.transaction()?; + assert!(named.expected.as_ref().unwrap().write_lease_required); + named.target = "odin".into(); + let error = stateful + .world + .engine + .authenticate_first_odin_warming_presence(&named, "warming-probe", 1, 2, &[1]) + .expect_err("a stateful target named odin that does not provide the rendezvous"); + assert!( + format!("{error:#}").contains("reserved for a stateful Odin incarnation"), + "{error:#}" + ); + Ok(()) + } + // ----------------------------------------------------------------- // A bad answer is an error. Only silence waits. // ----------------------------------------------------------------- diff --git a/src/drivers.rs b/src/drivers.rs index 749ac22..edb4699 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8350,7 +8350,7 @@ mod tests { "a body that is not CultNet", ); assert_refused( - challenge(scripted_peer(|_| b"HTTP/1.1 200 OK + challenge(scripted_peer(|_| b"HTTP/1.1 200 OK Content-Le".to_vec()), |_| {}), "a response that never finished its headers", ); @@ -8391,6 +8391,44 @@ Content-Le".to_vec()), |_| {}), ); } + /// The candidate path renders the Expected first, and rendering parses the + /// same endpoints, so it never reaches the parse in the challenge. The + /// stable path does not render: a bound stable endpoint that is not a plain + /// host and port is caught by the challenge's own parse, and it is a + /// configuration error, not a listener that has not answered yet. + #[test] + fn a_stable_endpoint_that_does_not_parse_is_an_error_not_a_wait() { + let temp = tempfile::tempdir().expect("a temporary directory"); + let driver = NginxRouteDriver::new(RouteBinding { + driver: RouteDriver::NginxStreamTcp, + route_id: "service".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + private_host: "127.0.0.1".into(), + private_port_start: 17998, + private_port_end: 17998, + config_path: temp.path().join("service.conf"), + reload_unit: "nginx.service".into(), + }); + let mut candidate = expected(); + candidate.route = Some(cultnet_rs::IdunnExpectedRoute { + route_id: "service".into(), + transport: "http".into(), + stable_endpoint: "http://127.0.0.1:17999".into(), + candidate_endpoint: "http://127.0.0.1:17998".into(), + }); + for endpoint in [ + "http://127.0.0.1", + "http://127.0.0.1:0", + "http://127.0.0.1:17999/path", + "tcp://127.0.0.1:17999", + ] { + assert_refused( + driver.request_runtime_presence_at(&candidate, "stable-probe", endpoint), + endpoint, + ); + } + } + #[test] fn route_connect_retries_connection_refused_until_listener_is_ready() -> Result<()> { let reservation = std::net::TcpListener::bind("127.0.0.1:0")?; From 86ee21dac5f65f2653356717080d60aa0c147a21 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 15:24:45 +0200 Subject: [PATCH 014/264] B3 fix: pin what cargo-mutants found unpinned, delete what it found redundant First cargo-mutants --in-diff pass over a61540b..aa1889b left 16 misses. Deleted: - the candidate challenge's own route-id check: rendering the Expected next holds the same binding, so it could never decide anything. - odin_can_observe in advance_warming: is_odin && !(!is_odin || observed) is is_odin && !observed; two of the misses were operators in that detour. Moved: which transactions are held for their evidence is now DeploymentTransaction::held_disagreement, so "finished or aborting work is never held" is a value with a test, not a local in resume_candidate. Pinned: the admitted-generation topology refresh (taken once, then not again), WarmingEvidence shape checks per source, the PresenceDisagrees text, the plan's readiness declaration needing the shared-infrastructure kind, TCP framed challenges (exact answer, bound, empty, non-CultNet, silence), RUDP silence, and ChallengeFailure's text and source. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 157 ++++++++++++++++++++++++++++++++++++++--- src/deployment_plan.rs | 4 +- src/drivers.rs | 124 +++++++++++++++++++++++++++++--- 3 files changed, 265 insertions(+), 20 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 84dd4d2..2152d5c 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1438,6 +1438,18 @@ impl DeploymentTransaction { .err() } + /// The disagreement that holds this transaction: evidence collected under + /// another class is held and reported, not advanced and not repaired, since + /// no step can make it the right kind and running one anyway only fails it + /// every tick. What is finished, or being finished (a completion, or an + /// abort under way), never reads the class, so it is left to run. + fn held_disagreement(&self) -> Option { + let finishing = self.completion.is_some() + || self.pre_fencing_abort.is_some() + || self.post_fencing_abort.is_some(); + self.readiness_disagreement().filter(|_| !finishing) + } + fn validate(&self) -> Result<()> { ensure!( self.schema_version == DEPLOYMENT_TRANSACTION_SCHEMA, @@ -4530,14 +4542,7 @@ impl Engine { { return Ok(false); } - // Evidence collected under another class is held and reported, not - // advanced and not repaired: no step can make it the right kind, and - // running one anyway only fails it every tick. An abort already under - // way does not read the class, so it is left to finish. - let aborting = current.value.completion.is_some() - || current.value.pre_fencing_abort.is_some() - || current.value.post_fencing_abort.is_some(); - if let Some(disagreement) = current.value.readiness_disagreement().filter(|_| !aborting) { + if let Some(disagreement) = current.value.held_disagreement() { self.note_fault("holds transaction", id, &anyhow::Error::new(disagreement)); return Ok(false); } @@ -5585,8 +5590,7 @@ impl Engine { } else { None }; - let odin_can_observe = !is_odin || odin_observation.is_some(); - let odin_observes_itself = is_odin && !odin_can_observe; + let odin_observes_itself = is_odin && odin_observation.is_none(); if odin_observes_itself { ensure!( expected.write_lease_required, @@ -10384,6 +10388,59 @@ mod tests { Ok(()) } + #[test] + fn warming_evidence_is_shape_checked_whatever_its_source() { + let good = |evidence: &ReadinessEvidence| match evidence { + ReadinessEvidence::RouteProof { evidence } => evidence.clone(), + ReadinessEvidence::OdinCorrelated { .. } => unreachable!("a route-proof fixture"), + }; + let presence = good(&route_proof_evidence()); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: presence.clone() }, + WarmingEvidence::RouteProofDirect { evidence: presence.clone() }, + WarmingEvidence::OdinTopology { evidence: topology(3, 1) }, + ] { + warming.validate_shape().expect("well-formed evidence"); + } + let mut damaged = presence; + damaged.canonical_bytes.push(1); + for warming in [ + WarmingEvidence::FirstOdinDirect { evidence: damaged.clone() }, + WarmingEvidence::RouteProofDirect { evidence: damaged }, + WarmingEvidence::OdinTopology { + evidence: TopologyEvidence { + canonical_sha256: "sha256:not-the-digest".into(), + ..topology(3, 1) + }, + }, + ] { + assert!(warming.validate_shape().is_err(), "{warming:?}"); + } + } + + #[test] + fn a_presence_disagreement_names_each_disagreement() { + let shortfall = PresenceDisagrees { + disagreements: vec![ + OdinTopologyDisagreement { + code: "capacity-below-minimum".into(), + expected: Some("2".into()), + observed: Some("1".into()), + }, + OdinTopologyDisagreement { + code: "state".into(), + expected: None, + observed: None, + }, + ], + }; + assert_eq!( + shortfall.to_string(), + "route proof answered with a runtime that disagrees with current authority: \ + capacity-below-minimum (expected 2, observed 1); state (expected none, observed none)" + ); + } + fn route_proof_evidence() -> ReadinessEvidence { let canonical_bytes = vec![7, 7, 7]; ReadinessEvidence::RouteProof { @@ -11925,6 +11982,51 @@ mod tests { .clone()) } + #[test] + fn an_admitted_odin_correlated_generation_takes_a_newer_odin_reading_once() -> Result<()> { + let world = EngineFixture::with_workload(Arc::new(StillWorkload))?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let first = seeded_sequence(None) + 1; + odin_reports_ready(&world, &seeded, first)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + assert!(world.engine.retire_one_terminal_transaction()?); + let admitted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("commit wrote no admitted generation")? + .value + .clone(); + assert_eq!(admitted.readiness(), Ok(ReadinessClass::OdinCorrelated)); + assert_eq!(admitted.odin_publisher_sequence_cursor, first); + + // Nothing newer than what is held: nothing is written. + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + let envelope = current.envelope.clone(); + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + assert_eq!(incumbent_envelope(&world)?, envelope); + + // A newer reading is adopted, once. + odin_republishes(&world, &seeded, first + 4)?; + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(world.engine.refresh_admitted_topology(&snapshot, current)?); + let refreshed = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no generation")? + .value + .clone(); + assert_eq!(refreshed.odin_publisher_sequence_cursor, first + 4); + assert_eq!( + refreshed.latest_odin_observation.as_ref().map(|latest| latest.publisher_sequence), + Some(first + 4) + ); + assert_eq!(refreshed.ready.odin().map(|ready| ready.publisher_sequence), Some(first + 4)); + let snapshot = ControlSnapshot::read(&world.state_store)?; + let current = snapshot.admitted_for("service").context("no generation")?; + assert!(!world.engine.refresh_admitted_topology(&snapshot, current)?); + Ok(()) + } + /// Change the incumbent in the store, compare-and-swap. fn edit_incumbent( world: &EngineFixture, @@ -15086,6 +15188,41 @@ mod tests { Ok(()) } + #[test] + fn only_a_transaction_that_still_reads_its_class_is_held_for_its_evidence() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + let at_routing = routed.transaction()?; + // What pre-B3 Idunn would have collected: an Odin receipt. + let mut legacy = at_routing.clone(); + legacy.ready = Some(ReadinessEvidence::OdinCorrelated { + evidence: odin_receipt_for(&routed, &at_routing, 9)?, + }); + let disagreement = legacy.readiness_disagreement(); + assert!(disagreement.is_some()); + assert_eq!(legacy.held_disagreement(), disagreement); + + // Finished work, and an abort under way, never read the class. + let mut completed = legacy.clone(); + completed.completion = Some(TransactionCompletion::Admitted { + generation_id: "generation-1".into(), + }); + assert_eq!(completed.held_disagreement(), None); + let mut aborting = legacy.clone(); + aborting.pre_fencing_abort = Some(PreFencingAbort { + error: "operator abort".into(), + candidate_cleanup: CleanupEvidence::Pending, + topology_reconciliation: CleanupEvidence::Pending, + source_cleanup: CleanupEvidence::Pending, + }); + assert_eq!(aborting.held_disagreement(), None); + let mut aborting = legacy; + aborting.post_fencing_abort = Some(post_fencing_abort_intent(&aborting, "operator abort")); + assert_eq!(aborting.held_disagreement(), None); + Ok(()) + } + #[test] fn a_stored_record_that_declares_no_way_to_prove_readiness_is_held() -> Result<()> { let world = EngineFixture::new()?; diff --git a/src/deployment_plan.rs b/src/deployment_plan.rs index 7169601..5210448 100644 --- a/src/deployment_plan.rs +++ b/src/deployment_plan.rs @@ -1585,12 +1585,14 @@ nodes = ["yggdrasil"] let provides_odin = |recipe: &str| { recipe.replace("capability = \"service.runtime\"", "capability = \"odin.verse-rendezvous\"") }; - let cases: [(&str, String, &str, Option, Result); 5] = [ + let cases: [(&str, String, &str, Option, Result); 6] = [ ("depends on Odin, routed", RECIPE.to_owned(), BINDING, Some(18001), Ok(ReadinessClass::OdinCorrelated)), ("routed, no Odin", without_odin.clone(), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)), ("unrouted, no Odin", unrouted(&without_odin), &unrouted_binding, None, Err(())), ("provides Odin, unrouted", provides_odin(&unrouted(&without_odin)), &unrouted_binding, None, Ok(ReadinessClass::OdinSelf)), ("depends on Odin, unrouted", unrouted(RECIPE), &unrouted_binding, None, Ok(ReadinessClass::OdinCorrelated)), + // Only the shared-infrastructure declaration is the declaration. + ("requires the Odin capability, routed", RECIPE.replace("kind = \"shared-infrastructure\"", "kind = \"required\""), BINDING, Some(18001), Ok(ReadinessClass::RouteProof)), ]; for (name, recipe, binding, port, outcome) in cases { let providers = [ready_odin_provider("odin", "odin-yggdrasil", 1)]; diff --git a/src/drivers.rs b/src/drivers.rs index edb4699..495fc86 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5468,10 +5468,7 @@ impl NginxRouteDriver { .as_ref() .context("candidate challenge has no Expected route") .map_err(refused)?; - ensure_or_refuse( - route.route_id == self.binding.route_id, - "candidate challenge binding differs from Expected", - )?; + // Rendering holds the binding to the Expected route, id and endpoints. self.render(expected).map_err(refused)?; self.request_runtime_presence_at(expected, message_id, &route.candidate_endpoint) } @@ -8205,20 +8202,31 @@ mod tests { fn challenge( port: u16, adjust: impl FnOnce(&mut IdunnExpectedIncarnationRecord), + ) -> Result { + challenge_over("http", RouteDriver::NginxStreamTcp, port, adjust) + } + + /// The same challenge over another transport: `transport` names both the + /// Expected's transport and the endpoint scheme. + fn challenge_over( + transport: &str, + driver: RouteDriver, + port: u16, + adjust: impl FnOnce(&mut IdunnExpectedIncarnationRecord), ) -> Result { let temp = tempfile::tempdir().expect("a temporary directory"); let mut candidate = expected(); candidate.route = Some(cultnet_rs::IdunnExpectedRoute { route_id: "service".into(), - transport: "http".into(), - stable_endpoint: "http://127.0.0.1:17999".into(), - candidate_endpoint: format!("http://127.0.0.1:{port}"), + transport: transport.into(), + stable_endpoint: format!("{transport}://127.0.0.1:17999"), + candidate_endpoint: format!("{transport}://127.0.0.1:{port}"), }); adjust(&mut candidate); NginxRouteDriver::new(RouteBinding { - driver: RouteDriver::NginxStreamTcp, + driver, route_id: "service".into(), - stable_endpoint: "http://127.0.0.1:17999".into(), + stable_endpoint: format!("{transport}://127.0.0.1:17999"), private_host: "127.0.0.1".into(), private_port_start: port, private_port_end: port, @@ -8227,6 +8235,104 @@ mod tests { }) .request_candidate_runtime_presence(&candidate, "candidate-probe") } + /// A TCP peer that reads one framed snapshot request and writes back the + /// bytes `reply` builds from the challenge id, verbatim. + fn framed_peer(reply: impl FnOnce(&str) -> Vec + Send + 'static) -> u16 { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); + let port = listener.local_addr().expect("a bound port").port(); + thread::spawn(move || { + let Ok((mut stream, _)) = listener.accept() else { + return; + }; + let mut header = [0_u8; 4]; + if stream.read_exact(&mut header).is_err() { + return; + } + let mut request = vec![0_u8; u32::from_be_bytes(header) as usize]; + if stream.read_exact(&mut request).is_err() { + return; + } + let Ok(CultNetMessage::SnapshotRequest { message_id, .. }) = + decode_cultnet_message_from_slice(&request, CultNetWireContract::CultNetSchemaV0) + else { + return; + }; + let _ = stream.write_all(&reply(&message_id)); + }); + port + } + + /// The body of an `honest` HTTP answer, framed for TCP. + fn framed_honest(message_id: &str) -> Vec { + let response = honest(message_id); + let body = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|end| &response[end + 4..]) + .expect("an HTTP answer"); + encode_frame(body).expect("a frame") + } + + fn tcp_challenge(port: u16) -> Result { + challenge_over("tcp", RouteDriver::NginxStreamTcp, port, |_| {}) + } + + #[test] + fn a_tcp_challenge_takes_the_exact_framed_answer_and_waits_only_on_silence() { + let answer = tcp_challenge(framed_peer(framed_honest)).expect("the exact answer"); + assert_eq!(answer.message_id, "candidate-probe"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + + assert_silent(tcp_challenge(framed_peer(|_| Vec::new())), "a peer that hangs up"); + assert_silent( + tcp_challenge(framed_peer(|_| vec![0, 0, 0, 9, 1])), + "a frame that never finishes", + ); + + // A frame that claims more than the observation bound, or nothing, is + // an answer that is wrong, not one that is late. + assert_refused( + tcp_challenge(framed_peer(|_| { + ((ROUTE_SNAPSHOT_MAX_BYTES + 1) as u32).to_be_bytes().to_vec() + })), + "a frame over the bound", + ); + assert_refused( + tcp_challenge(framed_peer(|_| 0_u32.to_be_bytes().to_vec())), + "an empty frame", + ); + assert_refused( + tcp_challenge(framed_peer(|_| encode_frame(b"not cultnet").expect("a frame"))), + "a frame that is not CultNet", + ); + } + + #[test] + fn a_rudp_challenge_that_nothing_answers_is_silence() { + let socket = std::net::UdpSocket::bind("127.0.0.1:0").expect("a free port"); + let port = socket.local_addr().expect("a bound port").port(); + // Bound but never read: nothing answers, and nothing refuses either. + assert_silent( + challenge_over("rudp", RouteDriver::NginxStreamUdp, port, |_| {}), + "a rudp peer that never answers", + ); + drop(socket); + } + + /// A challenge that cannot be made says why, whatever the message. + #[test] + fn a_challenge_failure_shows_and_chains_its_cause() { + for failure in [ + silent(anyhow!("cause of silence")), + refused(anyhow!("cause of refusal")), + ] { + let shown = failure.to_string(); + assert!(shown.starts_with("cause of "), "{shown}"); + let source = std::error::Error::source(&failure).expect("the underlying error"); + assert_eq!(source.to_string(), shown); + } + } + fn assert_silent(outcome: Result, why: &str) { assert!( From 7dc3a4bb8c438afd9820843d36e5c76b13a6ba42 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:00:39 +0200 Subject: [PATCH 015/264] Record the live store decode, B3 pass 2, and the ship sequence Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 1406316..b1b09aa 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -37,6 +37,33 @@ transactions of routed targets with no Odin declaration (operator approved, 2026 - Still open, for a fresh Hands: three survivors (boot re-proof by tag, config error as silence, first-Odin check by name) and `cargo mutants --in-diff`. +**Live store, read-only decode of a copy, 2026-09-30 14:56 (operator-approved).** +- `control.cc` holds 5 envelopes, all `admitted_generation.v2` (pre-F0), and **no transactions**. +- `history.cc` holds 957 transaction v3 envelopes and 641 command v2 envelopes. +- Admitted generations, all on Odin receipts: + - ghostlight: routed, declares Odin, stateful. + - odin: routed rudp, provides the rendezvous. + - raven-muninn: unrouted, no declaration, stateless. B3 class `Undeclared`, so **held**. + - streampixels-service: routed, declares Odin, stateful. + - streampixels-web: routed, declares Odin, needs `streampixels.service.api`. +- **Heimdall is not admitted.** It drops out of the ship window. Its declaring recipe still merges. Its + first deploy will hit Soul pass 2's F2: a routed, Odin-correlated target must pass the stable-route + proof, and Heimdall answers no challenge. Record that for its own deploy. + +**B3 pass 2 (Soul, 86ee21d):** safe for the web ship. One same-window blocker: +- **F1:** supervision mints a held continuity over a dead held generation, and that continuity owns the + target, so the declaring redeploy can never freeze. The fix is in Hands. +- F5, bad answers read as Silent over HTTP keep-alive and TCP, is in the same batch. + +**Ship sequence:** +1. B3 fix, then narrow Soul, then merge B3. +2. Merge StreamPixels `route/s3-c2` and the Muninn and Heimdall `route/b3-declare` recipes. +3. Build the Idunn release on its deploy path. +4. Back up `control.cc` and `history.cc`. F0's lift is irreversible. +5. Install Idunn. +6. `idunn up` raven-muninn with the declaring recipe. +7. `idunn up` streampixels (service, then web). + **Rulings, 2026-09-30:** - A stored record whose evidence disagrees with its declared class is **held and reported, never aborted** by Idunn. B5's deadlines or the operator resolve it. From 477927509ac8cd65f741cf069e35b48296322bc3 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:13:36 +0200 Subject: [PATCH 016/264] B3 fix: a held generation mints no continuity, and nothing yields to a held record A held, undeclared generation that died reached the continuity mint, which copied the undeclared Expected, was held at once, and owned the target for good: cancel refuses continuity, so the declaring redeploy could never freeze. Supervision now reports a dead held generation once and leaves it down and its target free. With no continuity minted over it, a declaring redeploy in Warming no longer yields to one either. The yield path also aborted a pre-fence Deploy blocker without reading its hold, which is Idunn aborting a held record. It now leaves a held blocker alone. Co-Authored-By: Claude Sonnet 5 --- src/control_plane.rs | 208 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/src/control_plane.rs b/src/control_plane.rs index 2152d5c..4bd6b59 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -4945,6 +4945,21 @@ impl Engine { } let workload_error = operational_error .context("admitted operational state has neither observation nor error")?; + // A held generation is a record the recipe never declared + // readiness for. Idunn reports it and does not decide for it: a + // continuity minted over it copies the undeclared Expected, is + // held at once, and owns the target forever, so the declaring + // redeploy that would clear the hold could never freeze. The dead + // held target stays down and free. Nothing yields to it either: + // there is no continuity to yield to. + if current.value.readiness().is_err() { + self.note_fault( + "holds a dead admitted generation down; declare readiness in its recipe and redeploy", + &format!("generation-down:{}", current.value.target), + &anyhow!("admitted generation {} is not running", current.value.generation_id), + ); + continue; + } // Continuity restarts a release; it cannot repair one. When the // admitted release will not start, rescheduling it forever keeps // the target permanently occupied -- and a target with a live @@ -5001,6 +5016,8 @@ impl Engine { && refused_restarts < CONTINUITY_RESTART_ATTEMPTS && blocker.value.phase < DeploymentPhase::Fencing && blocker.value.pre_fencing_abort.is_none() + // A held record is reported and never aborted by Idunn. + && blocker.value.held_disagreement().is_none() { self.begin_pre_fencing_abort( blocker, @@ -13729,6 +13746,197 @@ mod tests { Ok(()) } + // --------------------------------------------------------------------- + // A held record is reported and never decided for. Supervision mints no + // continuity over a held generation, and nothing yields to a held record. + // --------------------------------------------------------------------- + + /// Rewrite the world's admitted generation into what a pre-B3 binary left + /// behind: no Odin declaration, admitted on Odin receipts. + fn undeclare_incumbent(world: &EngineFixture) -> Result { + let now = now_millis()?; + let mut failure = None; + let next = edit_incumbent(world, |generation| { + generation.expected.dependencies.clear(); + match IdunnRuntimeActivationLaunch::issue( + &generation.expected, + generation.activation.runtime_instance_id.clone(), + now, + &world.engine.idunn_signer, + ) { + Ok(launch) => generation.activation = launch.activation().clone(), + Err(error) => failure = Some(error), + } + })?; + if let Some(error) = failure { + return Err(error); + } + ControlSnapshot::read(&world.state_store).context("store with the undeclared generation")?; + assert!(matches!(next.readiness(), Err(ReadinessDisagreement::Undeclared(_)))); + Ok(next) + } + + /// Queue the operator's declaring redeploy of `service`. + fn queue_declaring_redeploy(world: &EngineFixture, command_id: &str) -> Result<()> { + use crate::deployment_plan::tests::BINDING; + let (binding_head, binding_tail) = BINDING.split_once("[route]").context("route")?; + let binding = format!( + "{binding_head}[brakes]{}", + binding_tail.split_once("[brakes]").context("brakes")?.1 + ); + std::fs::create_dir_all(world.root.join("bindings"))?; + std::fs::write(world.root.join("bindings/service.toml"), binding)?; + let command = DeploymentCommand { + schema_version: DEPLOYMENT_COMMAND_SCHEMA.into(), + command_id: command_id.into(), + kind: CommandKind::Deploy, + selector: "service".into(), + requested_by: "operator".into(), + requested_at_unix_millis: now_millis()?, + }; + assert!(SingleFileMessagePackBackingStore::new(&world.state_store).compare_exchange( + &[CultCacheExpectedEnvelope { + r#type: DeploymentCommand::TYPE.into(), + key: command.command_id.clone(), + current: None, + }], + &[command_envelope(&command, command.requested_at_unix_millis)?], + )?); + Ok(()) + } + + /// Every transaction the store knows: live, then archived. + fn every_transaction(world: &EngineFixture) -> Result> { + let mut all = ControlSnapshot::read(&world.state_store)? + .transactions + .into_iter() + .map(|stored| stored.value) + .collect::>(); + all.extend(read_history_transactions(&world.state_store)); + Ok(all) + } + + fn is_continuity_over(transaction: &DeploymentTransaction, held: &AdmittedGeneration) -> bool { + transaction.command_kind == CommandKind::Continuity + && transaction.incumbent_generation_id.as_deref() == Some(held.generation_id.as_str()) + } + + #[test] + fn a_held_generation_that_dies_mints_no_continuity_and_leaves_its_target_free() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + world.engine.validate_durable_authority(&ControlSnapshot::read(&world.state_store)?)?; + + workload.kill(); + for _ in 0..4 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + assert!( + ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), + "supervision minted a transaction over a held generation" + ); + // Reported, once per state change: the same text is not offered again. + let reports = world.engine.fault_reports.lock().unwrap(); + let down = reports.get("generation-down:service").context("the dead hold is not reported")?; + let last = down.last.lock().unwrap().clone(); + assert!(last.is_some()); + assert!(down.offer(last).is_none()); + drop(reports); + + // The target is free: the declaring redeploy freezes and is not + // displaced by a continuity. + queue_declaring_redeploy(&world, "up-service-declared")?; + for _ in 0..8 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held generation" + ); + assert!( + all.iter().any(|transaction| transaction.command_id == "up-service-declared"), + "the declaring redeploy never froze" + ); + Ok(()) + } + + #[test] + fn a_declaring_redeploy_is_not_yielded_to_a_continuity_when_the_held_incumbent_dies() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + admit_incumbent(&world)?; + let held = undeclare_incumbent(&world)?; + let redeploy = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&held))?; + assert_eq!(redeploy.held_disagreement(), None); + workload.kill(); + for _ in 0..40 { + let _ = world.engine.run_scheduler_tick(); + } + let all = every_transaction(&world)?; + assert!( + all.iter().all(|transaction| !is_continuity_over(transaction, &held)), + "a continuity was minted over the held incumbent" + ); + let redeploy = all + .iter() + .find(|transaction| transaction.transaction_id == redeploy.transaction_id) + .context("the redeploy is nowhere")?; + // The fixture's one workload is the candidate too, so the redeploy may + // fail for its own reasons; it is never yielded away. + assert!( + redeploy + .pre_fencing_abort + .as_ref() + .is_none_or(|abort| !abort.error.contains("yielded to continuity")), + "the redeploy yielded: {:?}", + redeploy.pre_fencing_abort + ); + Ok(()) + } + + #[test] + fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + // The incumbent is declared; the Deploy in front of it is the held one. + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + let mut bare = seeded.clone(); + // Undeclare the Expected and re-derive what is bound to its digest. + let expected = bare.expected.as_mut().context("no expected")?; + expected.dependencies.clear(); + bare.expected_publication_sha256 = Some(expected.canonical_sha256()?); + let activation = IdunnRuntimeActivationLaunch::issue( + expected, + seeded.activation.as_ref().context("no activation")?.runtime_instance_id.clone(), + now_millis()?, + &world.engine.idunn_signer, + )? + .activation() + .clone(); + bare.activation_publication_sha256 = Some(activation.canonical_sha256()?); + bare.activation = Some(activation); + bare.validate()?; + let stored = resident(&world)?; + replace_transaction(&world.state_store, &stored, &bare)?; + assert!(bare.held_disagreement().is_some()); + + workload.kill(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; + } + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, seeded.transaction_id); + assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); + assert!(after.completion.is_none()); + Ok(()) + } + // --------------------------------------------------------------------- // Route-proof readiness: a routed target that declares no Odin dependency // is admitted by Idunn's own challenges. Odin is never read. From 2f22dcac96c6c14d66c6f781a19d6ef1b3ca81cd Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:13:37 +0200 Subject: [PATCH 017/264] B3 fix: read a route answer by its framing, and judge a stopped peer one way The HTTP challenge read to end of stream, so a peer that answered in full and kept the socket open (or answered 404 and kept it open) read as silence. It now reads by Content-Length or chunked framing, refuses any non-2xx status, and refuses bytes that are not a CultNet frame. One rule for both transports: a peer that sent no byte is silent; a peer that sent some bytes and stopped is refused. A truncated TCP frame used to read as silence; that pinned test now expects a refusal. Co-Authored-By: Claude Sonnet 5 --- src/drivers.rs | 440 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 374 insertions(+), 66 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 495fc86..2429c9a 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fs::{self, OpenOptions}; -use std::io::{BufRead, ErrorKind, Read, Write}; +use std::io::{BufRead, BufReader, ErrorKind, Read, Write}; use std::net::{SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; @@ -700,6 +700,7 @@ const ROUTE_CONNECT_RETRY_WINDOW: Duration = Duration::from_secs(2); const ROUTE_CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(25); const ROUTE_SNAPSHOT_MAX_BYTES: usize = 1024 * 1024; const ROUTE_HTTP_MAX_HEADER_BYTES: usize = 32 * 1024; +const ROUTE_HTTP_CHUNK_LINE_BYTES: usize = 128; const ROUTE_HTTP_SNAPSHOT_PATH: &str = "/cultnet/snapshot"; impl RoutePreflightReceipt { @@ -5607,6 +5608,70 @@ fn ensure_or_refuse(condition: bool, message: &'static str) -> Result<(), Challe } } +/// One answer as it arrives, over either transport. A peer that stops is judged +/// the same way on both: no byte at all is silence (a process still starting +/// does that), and anything short of a whole answer after some bytes is a bad +/// answer. Waiting does not finish an answer that has begun and stopped. +struct Answer { + source: R, + seen: usize, +} + +impl Answer { + fn new(source: R) -> Self { + Self { source, seen: 0 } + } + + fn stopped(&self, what: &str) -> ChallengeFailure { + let error = anyhow!("stable CultNet route stopped {what}"); + if self.seen == 0 { + silent(error) + } else { + refused(error) + } + } + + fn read_some(&mut self, buffer: &mut [u8], what: &str) -> Result { + loop { + match self.source.read(buffer) { + Ok(0) => return Err(self.stopped(what)), + Ok(read) => { + self.seen += read; + return Ok(read); + } + Err(error) if error.kind() == ErrorKind::Interrupted => {} + Err(_) => return Err(self.stopped(what)), + } + } + } + + fn read_all(&mut self, buffer: &mut [u8], what: &str) -> Result<(), ChallengeFailure> { + let mut filled = 0; + while filled < buffer.len() { + filled += self.read_some(&mut buffer[filled..], what)?; + } + Ok(()) + } + + /// One CRLF-terminated line, without its terminator. + fn read_line(&mut self, bound: usize, what: &str) -> Result, ChallengeFailure> { + let mut line = Vec::new(); + loop { + let mut byte = [0_u8]; + self.read_all(&mut byte, what)?; + line.push(byte[0]); + if line.ends_with(b"\r\n") { + line.truncate(line.len() - 2); + return Ok(line); + } + ensure_or_refuse( + line.len() <= bound, + "stable CultNet HTTP response line exceeds its bound", + )?; + } + } +} + fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, ChallengeFailure> { let mut stream = connect_route_socket(target) .with_context(|| format!("connecting stable CultNet TCP route {target}")) @@ -5622,21 +5687,16 @@ fn request_tcp_snapshot(target: SocketAddr, payload: &[u8]) -> Result, C .and_then(|()| stream.flush()) .map_err(silent)?; + let mut answer = Answer::new(stream); let mut header = [0_u8; 4]; - stream - .read_exact(&mut header) - .context("reading stable CultNet TCP response frame") - .map_err(silent)?; + answer.read_all(&mut header, "before its response frame header ended")?; let length = u32::from_be_bytes(header) as usize; ensure_or_refuse( (1..=ROUTE_SNAPSHOT_MAX_BYTES).contains(&length), "stable CultNet TCP response exceeds the route observation bound", )?; let mut response = vec![0_u8; length]; - stream - .read_exact(&mut response) - .context("reading stable CultNet TCP response payload") - .map_err(silent)?; + answer.read_all(&mut response, "before its response frame ended")?; Ok(response) } @@ -5660,79 +5720,124 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, .and_then(|()| stream.flush()) .map_err(silent)?; - let maximum = ROUTE_HTTP_MAX_HEADER_BYTES - .checked_add(ROUTE_SNAPSHOT_MAX_BYTES) - .context("route HTTP response bound overflow") + // The response is read by its own framing, never to end of stream: a peer + // that ignores `Connection: close` and keeps the socket open has still + // answered in full. + let mut answer = Answer::new(BufReader::new(stream)); + let mut head_bytes = 0_usize; + let mut head_line = |answer: &mut Answer>| { + let line = answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, "before its headers ended")?; + head_bytes += line.len() + 2; + ensure_or_refuse( + head_bytes <= ROUTE_HTTP_MAX_HEADER_BYTES, + "stable CultNet HTTP response headers exceed their bound", + )?; + Ok::<_, ChallengeFailure>(line) + }; + let status = head_line(&mut answer)?; + let status = std::str::from_utf8(&status) + .context("stable CultNet HTTP response status is not UTF-8") .map_err(refused)?; - let mut response = Vec::new(); - (&mut stream) - .take((maximum + 1) as u64) - .read_to_end(&mut response) - .context("reading stable CultNet HTTP response") - .map_err(silent)?; - if response.is_empty() { - return Err(silent(anyhow!( - "stable CultNet HTTP route closed the connection without answering" + if !is_http_success(status) { + return Err(refused(anyhow!( + "stable CultNet HTTP route rejected the snapshot challenge: {status}" ))); } - parse_http_snapshot(&response, maximum).map_err(refused) -} -/// The body of an HTTP snapshot response that arrived, or why what arrived is -/// not one. -fn parse_http_snapshot(response: &[u8], maximum: usize) -> Result> { - ensure!( - response.len() <= maximum, - "stable CultNet HTTP response exceeds the route observation bound" - ); - let header_end = response - .windows(4) - .position(|window| window == b"\r\n\r\n") - .map(|index| index + 4) - .context("stable CultNet HTTP response has no header boundary")?; - ensure!( - header_end <= ROUTE_HTTP_MAX_HEADER_BYTES, - "stable CultNet HTTP response headers exceed their bound" - ); - let headers = std::str::from_utf8(&response[..header_end]) - .context("stable CultNet HTTP response headers are not UTF-8")?; - let mut lines = headers.split("\r\n"); - let status = lines - .next() - .context("stable CultNet HTTP response has no status")?; - ensure!( - matches!(status, "HTTP/1.1 200 OK" | "HTTP/1.0 200 OK"), - "stable CultNet HTTP route rejected the snapshot challenge: {status}" - ); let mut content_length = None; - for line in lines.filter(|line| !line.is_empty()) { + let mut chunked = false; + loop { + let line = head_line(&mut answer)?; + if line.is_empty() { + break; + } + let line = std::str::from_utf8(&line) + .context("stable CultNet HTTP response headers are not UTF-8") + .map_err(refused)?; let (name, value) = line .split_once(':') - .context("stable CultNet HTTP response contains a malformed header")?; + .context("stable CultNet HTTP response contains a malformed header") + .map_err(refused)?; if name.eq_ignore_ascii_case("transfer-encoding") { - bail!("stable CultNet HTTP response uses unsupported transfer encoding"); + ensure_or_refuse( + value.trim().eq_ignore_ascii_case("chunked"), + "stable CultNet HTTP response uses unsupported transfer encoding", + )?; + chunked = true; } if name.eq_ignore_ascii_case("content-length") { - ensure!( + ensure_or_refuse( content_length.is_none(), - "stable CultNet HTTP response repeats Content-Length" - ); + "stable CultNet HTTP response repeats Content-Length", + )?; content_length = Some( value .trim() .parse::() - .context("stable CultNet HTTP Content-Length is invalid")?, + .context("stable CultNet HTTP Content-Length is invalid") + .map_err(refused)?, ); } } - let content_length = - content_length.context("stable CultNet HTTP response has no Content-Length")?; - ensure!( - content_length <= ROUTE_SNAPSHOT_MAX_BYTES - && response.len().saturating_sub(header_end) == content_length, - "stable CultNet HTTP response body differs from its bounded Content-Length" - ); - Ok(response[header_end..].to_vec()) + + let body = "before its body ended"; + match (content_length, chunked) { + (Some(length), false) => { + ensure_or_refuse( + length <= ROUTE_SNAPSHOT_MAX_BYTES, + "stable CultNet HTTP response body exceeds the route observation bound", + )?; + let mut response = vec![0_u8; length]; + answer.read_all(&mut response, body)?; + Ok(response) + } + (None, true) => { + let mut response = Vec::new(); + loop { + let size = answer.read_line(ROUTE_HTTP_CHUNK_LINE_BYTES, body)?; + let size = std::str::from_utf8(&size) + .ok() + .and_then(|size| size.split(';').next()) + .and_then(|size| usize::from_str_radix(size.trim(), 16).ok()) + .context("stable CultNet HTTP chunk size is invalid") + .map_err(refused)?; + if size == 0 { + // Trailers, if any, end at the empty line. + while !answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, body)?.is_empty() {} + return Ok(response); + } + ensure_or_refuse( + response.len().saturating_add(size) <= ROUTE_SNAPSHOT_MAX_BYTES, + "stable CultNet HTTP response body exceeds the route observation bound", + )?; + let start = response.len(); + response.resize(start + size, 0); + answer.read_all(&mut response[start..], body)?; + let mut end = [0_u8; 2]; + answer.read_all(&mut end, body)?; + ensure_or_refuse(&end == b"\r\n", "stable CultNet HTTP chunk is not terminated")?; + } + } + _ => Err(refused(anyhow!( + "stable CultNet HTTP response has no single body framing" + ))), + } +} + +/// A 2xx status line of HTTP/1.0 or HTTP/1.1. +fn is_http_success(status: &str) -> bool { + let Some(rest) = status + .strip_prefix("HTTP/1.1 ") + .or_else(|| status.strip_prefix("HTTP/1.0 ")) + else { + return false; + }; + let code = rest.as_bytes(); + code.len() >= 3 + && code[0] == b'2' + && code[1].is_ascii_digit() + && code[2].is_ascii_digit() + && (code.len() == 3 || code[3] == b' ') } /// nginx reload is signal-driven: `systemctl reload` can return before its @@ -8121,8 +8226,17 @@ mod tests { // --------------------------------------------------------------------- /// A peer that reads one snapshot request and writes back whatever `reply` - /// builds from the challenge id. Returns the port it listens on. + /// builds from the challenge id, then closes. Returns the port it listens on. fn scripted_peer(reply: impl FnOnce(&str) -> Vec + Send + 'static) -> u16 { + lingering_peer(reply, Duration::ZERO) + } + + /// `scripted_peer`, except the socket stays open for `linger` after the + /// answer: keep-alive, ignoring `Connection: close`. + fn lingering_peer( + reply: impl FnOnce(&str) -> Vec + Send + 'static, + linger: Duration, + ) -> u16 { let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("a free port"); let port = listener.local_addr().expect("a bound port").port(); thread::spawn(move || { @@ -8152,6 +8266,8 @@ mod tests { } }; let _ = stream.write_all(&reply(&message_id)); + let _ = stream.flush(); + thread::sleep(linger); }); port } @@ -8284,7 +8400,8 @@ mod tests { assert_eq!(answer.canonical_presence, [1, 2, 3]); assert_silent(tcp_challenge(framed_peer(|_| Vec::new())), "a peer that hangs up"); - assert_silent( + // Some bytes and then nothing is a bad answer, not a late one. + assert_refused( tcp_challenge(framed_peer(|_| vec![0, 0, 0, 9, 1])), "a frame that never finishes", ); @@ -8334,6 +8451,197 @@ mod tests { } + // --------------------------------------------------------------------- + // One rule for a peer that stops, on either transport: no byte at all is + // silence; some bytes and then nothing is a bad answer. An answer is read + // by its own framing, so a peer that keeps the socket open has still + // answered. + // --------------------------------------------------------------------- + + fn http_chunked(body: &[u8], chunk: usize) -> Vec { + let mut response = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n".to_vec(); + for piece in body.chunks(chunk) { + response.extend_from_slice(format!("{:x};ext=1\r\n", piece.len()).as_bytes()); + response.extend_from_slice(piece); + response.extend_from_slice(b"\r\n"); + } + response.extend_from_slice(b"0\r\nTrailer: x\r\n\r\n"); + response + } + + fn honest_body(message_id: &str) -> Vec { + let response = honest(message_id); + let end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .expect("an HTTP answer"); + response[end + 4..].to_vec() + } + + const LINGER: Duration = Duration::from_secs(6); + + #[test] + fn an_http_answer_is_read_by_its_framing_not_to_the_end_of_the_stream() { + let started = Instant::now(); + let answer = challenge(lingering_peer(honest, LINGER), |_| {}).expect("a whole answer"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + + let started = Instant::now(); + let answer = challenge( + lingering_peer(|id| http_chunked(&honest_body(id), 7), LINGER), + |_| {}, + ) + .expect("a chunked answer"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + } + + #[test] + fn an_http_status_that_is_not_success_is_refused_even_when_the_peer_lingers() { + for status in [ + "404 Not Found", + "301 Moved Permanently", + "503 Service Unavailable", + "199 Odd", + ] { + // The body is the honest answer: only the status refuses it. + let reply = move |id: &str| { + let body = honest_body(id); + let mut reply = + format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\n\r\n", body.len()) + .into_bytes(); + reply.extend_from_slice(&body); + reply + }; + let started = Instant::now(); + assert_refused(challenge(lingering_peer(reply, LINGER), |_| {}), status); + assert!(started.elapsed() < LINGER / 2, "{status} waited on the socket"); + } + } + + #[test] + fn only_a_two_hundred_series_http_1_status_line_is_success() { + for status in ["HTTP/1.1 200 OK", "HTTP/1.0 200 OK", "HTTP/1.1 204", "HTTP/1.1 299 x"] { + assert!(is_http_success(status), "{status}"); + } + for status in [ + "", + "HTTP/2 200 OK", + "HTTP/1.1 ", + "HTTP/1.1 20", + "HTTP/1.1 2x0 OK", + "HTTP/1.1 20x OK", + "HTTP/1.1 2000 OK", + "HTTP/1.1 300 OK", + "HTTP/1.1 404 Not Found", + "200 OK", + ] { + assert!(!is_http_success(status), "{status:?}"); + } + } + + #[test] + fn a_peer_that_sent_some_bytes_and_stopped_is_refused_on_either_transport() { + // No byte at all is silence, on both. + assert_silent(challenge(scripted_peer(|_| Vec::new()), |_| {}), "an HTTP peer with nothing to say"); + assert_silent(tcp_challenge(framed_peer(|_| Vec::new())), "a TCP peer with nothing to say"); + + // Some bytes and then nothing is a bad answer, wherever it stops. + let head = b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n"; + for (why, bytes) in [ + ("a status line that stops", b"HTTP/1.1 2".to_vec()), + ("headers that stop", b"HTTP/1.1 200 OK\r\nContent-Le".to_vec()), + ("a body that stops", [head.as_slice(), b"abc"].concat()), + ( + "a chunk that stops", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nab".to_vec(), + ), + ( + "a chunked body that never ends", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1\r\na\r\n".to_vec(), + ), + ] { + assert_refused(challenge(scripted_peer(move |_| bytes), |_| {}), why); + } + for (why, bytes) in [ + ("a 3-byte banner", b"OK\n".to_vec()), + ("a frame header that stops", vec![0, 0]), + ("a frame that stops", vec![0, 0, 0, 9, 1]), + ] { + assert_refused(tcp_challenge(framed_peer(move |_| bytes)), why); + } + } + + #[test] + fn an_http_answer_that_frames_its_body_wrongly_is_refused() { + for (why, reply) in [ + ( + "an unsupported transfer encoding", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: gzip\r\n\r\n".to_vec(), + ), + ( + "both framings", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Length: 1\r\n\r\n0\r\n\r\n" + .to_vec(), + ), + ("no framing", b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nabc".to_vec()), + ( + "a repeated Content-Length", + b"HTTP/1.1 200 OK\r\nContent-Length: 1\r\nContent-Length: 1\r\n\r\na".to_vec(), + ), + ( + "an invalid Content-Length", + b"HTTP/1.1 200 OK\r\nContent-Length: many\r\n\r\n".to_vec(), + ), + ( + "a Content-Length over the bound", + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", + ROUTE_SNAPSHOT_MAX_BYTES + 1 + ) + .into_bytes(), + ), + ("a malformed header", b"HTTP/1.1 200 OK\r\nno colon here\r\n\r\n".to_vec()), + ( + "an invalid chunk size", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\nzz\r\n".to_vec(), + ), + ( + "an unterminated chunk", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1\r\naXX0\r\n\r\n".to_vec(), + ), + ( + "chunks over the bound", + format!( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n{:x}\r\n", + ROUTE_SNAPSHOT_MAX_BYTES + 1 + ) + .into_bytes(), + ), + ( + "one header line over the bound", + [ + b"HTTP/1.1 200 OK\r\nX: ".to_vec(), + vec![b'a'; ROUTE_HTTP_MAX_HEADER_BYTES + 1], + ] + .concat(), + ), + ( + "headers over the bound together", + [ + b"HTTP/1.1 200 OK\r\n".to_vec(), + b"X: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n".repeat(600), + ] + .concat(), + ), + ("headers that are not UTF-8", b"HTTP/1.1 200 OK\r\nX: \xff\r\n\r\n".to_vec()), + ("a status that is not UTF-8", b"HTTP/1.1 \xff\r\n\r\n".to_vec()), + ] { + assert_refused(challenge(scripted_peer(move |_| reply), |_| {}), why); + } + } + fn assert_silent(outcome: Result, why: &str) { assert!( matches!(outcome, Err(ChallengeFailure::Silent(_))), From 3803725bf2a10311e99d3059d4007dcc2ce303ca Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:21:25 +0200 Subject: [PATCH 018/264] Record the Odin timeout diagnosis and the Idunn-first ship order Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index b1b09aa..bf77513 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -55,6 +55,20 @@ transactions of routed targets with no Odin declaration (operator approved, 2026 target, so the declaring redeploy can never freeze. The fix is in Hands. - F5, bad answers read as Silent over HTTP keep-alive and TCP, is in the same batch. +**Odin diagnosis (Eyes, read-only, 2026-09-30).** +- The 109 Odin route timeouts are self-inflicted. Live Idunn (`8ae00a1`, pre-S1) reloads nginx inside + `restore_admitted_membership` before every Odin challenge. The reload moves the reuseport UDP flow, so + the challenge misses. All 109 have a reload 3.05-3.12 s before them. +- 2,977 reloads in 6 h. 479 shutting-down nginx workers, because the stream has no `proxy_timeout`. +- The route timeouts do not block Ready correlation. +- Odin has crashed twice right after a reload. A stray RUDP packet with a foreign connection id escapes + `cultnet-rs` `rudp.rs:1217` (`require_connection`) and ends the process. Each crash costs about 75 s + without a lease. The CultLib fix is in Hands on `hands/rudp-stray-packets`. Odin takes it at its next pin + bump, after the ship. + +**Operator ruling: Idunn first.** Install the new Idunn (B3 carries S1), then watch for about 15 minutes that +reloads stop and Odin stays up. Only then redeploy raven-muninn and StreamPixels. + **Ship sequence:** 1. B3 fix, then narrow Soul, then merge B3. 2. Merge StreamPixels `route/s3-c2` and the Muninn and Heimdall `route/b3-declare` recipes. From d5669af3522da309f6d1da58facba5a5a2773b00 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 17:49:56 +0200 Subject: [PATCH 019/264] B3 fix: pin what cargo-mutants found unpinned in the framed route reader The header-size bound was tested only against answers that also stopped short, so it never had to hold on its own; the interrupted-read retry was exercised by no reader that could be interrupted. The header test now sends an honest, whole answer whose headers alone exceed the bound, and a scripted reader pins the retry and the silent/refused split at the Answer boundary. Co-Authored-By: Claude Sonnet 5 --- src/drivers.rs | 86 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 2429c9a..f06e019 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8627,14 +8627,6 @@ mod tests { ] .concat(), ), - ( - "headers over the bound together", - [ - b"HTTP/1.1 200 OK\r\n".to_vec(), - b"X: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n".repeat(600), - ] - .concat(), - ), ("headers that are not UTF-8", b"HTTP/1.1 200 OK\r\nX: \xff\r\n\r\n".to_vec()), ("a status that is not UTF-8", b"HTTP/1.1 \xff\r\n\r\n".to_vec()), ] { @@ -8642,6 +8634,84 @@ mod tests { } } + /// Every header line is small and the answer is honest and whole; only the + /// headers' total size refuses it. + #[test] + fn headers_that_are_over_their_bound_together_refuse_an_otherwise_honest_answer() { + let filler = b"X: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"; + let answer_with = |lines: usize| { + move |id: &str| { + let body = honest_body(id); + let mut reply = + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n", body.len()).into_bytes(); + reply.extend_from_slice(&filler.repeat(lines)); + reply.extend_from_slice(b"\r\n"); + reply.extend_from_slice(&body); + reply + } + }; + let lines = ROUTE_HTTP_MAX_HEADER_BYTES / filler.len(); + challenge(scripted_peer(answer_with(lines / 2)), |_| {}).expect("headers inside the bound"); + assert_refused( + challenge(scripted_peer(answer_with(lines * 2)), |_| {}), + "headers over the bound together", + ); + } + + /// A reader that fails `interruptions` times with `kind`, then serves `bytes`. + struct Flaky { + kind: ErrorKind, + interruptions: usize, + bytes: &'static [u8], + } + + impl Read for Flaky { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.interruptions > 0 { + self.interruptions -= 1; + return Err(self.kind.into()); + } + let served = self.bytes.len().min(buffer.len()); + buffer[..served].copy_from_slice(&self.bytes[..served]); + self.bytes = &self.bytes[served..]; + Ok(served) + } + } + + #[test] + fn an_interrupted_read_is_retried_and_any_other_failure_is_a_stopped_peer() { + let mut interrupted = Answer::new(Flaky { + kind: ErrorKind::Interrupted, + interruptions: 3, + bytes: b"abcd", + }); + let mut buffer = [0_u8; 4]; + interrupted.read_all(&mut buffer, "in the test").expect("the read is retried"); + assert_eq!(&buffer, b"abcd"); + + // Before any byte the peer is silent; after some, it is refused. + let mut failing = Answer::new(Flaky { + kind: ErrorKind::TimedOut, + interruptions: 1, + bytes: b"abcd", + }); + assert!(matches!( + failing.read_all(&mut buffer, "in the test"), + Err(ChallengeFailure::Silent(_)) + )); + let mut failing = Answer::new(Flaky { + kind: ErrorKind::TimedOut, + interruptions: 0, + bytes: b"ab", + }); + let mut wanted = [0_u8; 3]; + failing.read_all(&mut wanted[..2], "in the test").expect("two bytes"); + assert!(matches!( + failing.read_all(&mut wanted[2..], "in the test"), + Err(ChallengeFailure::Refused(_)) + )); + } + fn assert_silent(outcome: Result, why: &str) { assert!( matches!(outcome, Err(ChallengeFailure::Silent(_))), From 4b3b86d8633ff95db92461229f23a64720d2f463 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 18:19:07 +0200 Subject: [PATCH 020/264] B3 fix: pin the header bound at its edge and the chunked trailer loop Two more mutants survived: the header-size bound was only tested far from its edge, so doubling each line's cost went unseen; and the chunked trailer loop was only exercised with a trailer present, so inverting it went unseen. The bound is now tested one line inside and at its edge, and a chunked answer with no trailer is read from a peer that keeps the socket open. Co-Authored-By: Claude Sonnet 5 --- src/drivers.rs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index f06e019..78eb028 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8458,14 +8458,18 @@ mod tests { // answered. // --------------------------------------------------------------------- - fn http_chunked(body: &[u8], chunk: usize) -> Vec { + fn http_chunked(body: &[u8], chunk: usize, trailer: bool) -> Vec { let mut response = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n".to_vec(); for piece in body.chunks(chunk) { response.extend_from_slice(format!("{:x};ext=1\r\n", piece.len()).as_bytes()); response.extend_from_slice(piece); response.extend_from_slice(b"\r\n"); } - response.extend_from_slice(b"0\r\nTrailer: x\r\n\r\n"); + response.extend_from_slice(if trailer { + b"0\r\nTrailer: x\r\n\r\n" + } else { + b"0\r\n\r\n" + }); response } @@ -8487,14 +8491,16 @@ mod tests { assert_eq!(answer.canonical_presence, [1, 2, 3]); assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); - let started = Instant::now(); - let answer = challenge( - lingering_peer(|id| http_chunked(&honest_body(id), 7), LINGER), - |_| {}, - ) - .expect("a chunked answer"); - assert_eq!(answer.canonical_presence, [1, 2, 3]); - assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + for trailer in [true, false] { + let started = Instant::now(); + let answer = challenge( + lingering_peer(move |id| http_chunked(&honest_body(id), 7, trailer), LINGER), + |_| {}, + ) + .expect("a chunked answer"); + assert_eq!(answer.canonical_presence, [1, 2, 3]); + assert!(started.elapsed() < LINGER / 2, "the challenge waited on the socket"); + } } #[test] @@ -8651,9 +8657,10 @@ mod tests { } }; let lines = ROUTE_HTTP_MAX_HEADER_BYTES / filler.len(); - challenge(scripted_peer(answer_with(lines / 2)), |_| {}).expect("headers inside the bound"); + // Just inside the bound, then just outside it. + challenge(scripted_peer(answer_with(lines - 2)), |_| {}).expect("headers inside the bound"); assert_refused( - challenge(scripted_peer(answer_with(lines * 2)), |_| {}), + challenge(scripted_peer(answer_with(lines)), |_| {}), "headers over the bound together", ); } From e8b44878b76cf88ca01728d157e84e1f4eb07bfa Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 18:23:11 +0200 Subject: [PATCH 021/264] B3 fix: the over-bound header test must be over the bound The previous commit's over-bound case used floor(bound / line) lines, which can still fit under the bound once the status and Content-Length lines are counted; the answer was accepted and the test failed. It now sends one whole line more. Co-Authored-By: Claude Sonnet 5 --- src/drivers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 78eb028..89d5a3a 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -8657,10 +8657,10 @@ mod tests { } }; let lines = ROUTE_HTTP_MAX_HEADER_BYTES / filler.len(); - // Just inside the bound, then just outside it. + // Just inside the bound, then just outside it (a whole line over). challenge(scripted_peer(answer_with(lines - 2)), |_| {}).expect("headers inside the bound"); assert_refused( - challenge(scripted_peer(answer_with(lines)), |_| {}), + challenge(scripted_peer(answer_with(lines + 1)), |_| {}), "headers over the bound together", ); } From bf2c645a60b632c7999f9dde6c96041da462c392 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:35:16 +0200 Subject: [PATCH 022/264] Map: B3 fix batch Soul pass 3 and the held pre-fence cancel ruling Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index bf77513..54502d5 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -55,6 +55,24 @@ transactions of routed targets with no Odin declaration (operator approved, 2026 target, so the declaring redeploy can never freeze. The fix is in Hands. - F5, bad answers read as Silent over HTTP keep-alive and TCP, is in the same batch. +**B3 fix batch (`86ee21d..e8b4487`), Soul pass 3, 2026-09-30: merge with follow-ups.** Nothing here hurts a +live host on install. 294 passed, 2 ignored. `cargo mutants --in-diff`: 53 caught, 0 missed. +- **F1 holds.** There is one continuity mint site, and the hold check comes before it. A declaring redeploy + replaces a dead held incumbent end to end. +- **F5 holds.** Zero bytes is Silent. Any byte then a stall or stop is Refused. Non-2xx responses and + non-chunked Transfer-Encoding are Refused. +- **Follow-ups, in Hands on the same branch:** + - the P1b test never reaches the yield branch (mutant M7 survives); + - the chunked trailer loop is unbounded; + - `+N` lengths and whitespace before the colon are accepted; + - exact 1 MiB and one-byte-then-stop are unpinned; + - the `Silent` doc is stale; + - the dead-hold report goes to stderr only and never clears. + +**Ruling, 2026-09-30 (operator): cancel accepts a held pre-fence Deploy.** Idunn still never aborts a held +record on its own. An operator can discard a held Deploy that has not reached the fence with `idunn cancel`. +Nothing is fenced, so the incumbent is untouched. + **Odin diagnosis (Eyes, read-only, 2026-09-30).** - The 109 Odin route timeouts are self-inflicted. Live Idunn (`8ae00a1`, pre-S1) reloads nginx inside `restore_admitted_membership` before every Odin challenge. The reload moves the reuseport UDP flow, so From 137e120ce95ed3c214179f7e94a9b6d1867da892 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:36:54 +0200 Subject: [PATCH 023/264] B3 follow-up: cancel accepts a held pre-fence Deploy; test yield by direct supervision Operator ruling 2026-09-30 (F4): an operator may cancel a held Deploy that has not fenced. Nothing was fenced, so the incumbent is untouched; the pre-fencing abort releases the target. Idunn itself still never aborts a held record: the held_disagreement guard on the yield path is unchanged and pinned. The P1b test now calls supervise_one_admitted_generation directly with the redeploy in Warming over a dead held incumbent, with a positive control (the same redeploy over a dead non-held incumbent yields). Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 171 +++++++++++++++++++++++++++++++++---------- 1 file changed, 133 insertions(+), 38 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 4bd6b59..bba9e4a 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -2447,8 +2447,8 @@ fn parse_cancel(mut args: impl Iterator) -> Result { }) } -/// Withdraw queued work, or ask Idunn to clean up a live stateless candidate. -/// Stateful candidates past fencing cannot be cancelled because their prior +/// Withdraw queued work, a held deployment that has not fenced, or ask Idunn to +/// clean up a live stateless candidate. Stateful candidates past fencing cannot be cancelled because their prior /// writer may already be stopped and its state boundary needs explicit repair. fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> { let snapshot = ControlSnapshot::read(store_path)?; @@ -2471,12 +2471,17 @@ fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> "deployment command has multiple live transactions" ); ensure!( - cancel_is_safe_for_live_stateless(¤t.value), - "live deployment can only be cancelled after fencing when its candidate is stateless" + cancel_is_safe_for_live_stateless(¤t.value) + || cancel_is_safe_for_held_prefence(¤t.value), + "live deployment can only be cancelled before fencing when Idunn holds it, or after fencing when its candidate is stateless" ); let error = format!("cancelled by {requested_by}"); let mut next = current.value.clone(); - next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); + if cancel_is_safe_for_held_prefence(&next) { + next.pre_fencing_abort = Some(pre_fencing_abort_intent(&next, &error)); + } else { + next.post_fencing_abort = Some(post_fencing_abort_intent(&next, &error)); + } next.last_error = Some(error); next.updated_at_unix_millis = now_millis()?; replace_transaction(store_path, current, &next)?; @@ -2513,6 +2518,14 @@ fn cancel(store_path: &Path, command_id: &str, requested_by: &str) -> Result<()> Ok(()) } +/// A deployment Idunn holds before it fenced anything has touched nothing of +/// the incumbent, so an operator may withdraw it. Idunn never does so itself. +fn cancel_is_safe_for_held_prefence(transaction: &DeploymentTransaction) -> bool { + transaction.command_kind == CommandKind::Deploy + && transaction.phase < DeploymentPhase::Fencing + && transaction.held_disagreement().is_some() +} + fn cancel_is_safe_for_live_stateless(transaction: &DeploymentTransaction) -> bool { transaction.command_kind == CommandKind::Deploy && transaction.phase >= DeploymentPhase::Fencing @@ -7582,19 +7595,7 @@ impl Engine { current.value.pre_fencing_abort.is_none(), "pre-fencing abort intent is already durable" ); - let abort = PreFencingAbort { - error: truncate(&format!("{error:#}"), 2048), - candidate_cleanup: candidate_cleanup_requirement( - current.value.activation.is_some(), - current.value.workload.is_some(), - ), - topology_reconciliation: current.value.abort_topology_reconciliation(), - source_cleanup: if current.value.command_kind == CommandKind::Deploy { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - }, - }; + let abort = pre_fencing_abort_intent(¤t.value, &format!("{error:#}")); self.persist_same_phase(current, |next| { next.pre_fencing_abort = Some(abort); Ok(()) @@ -7987,6 +7988,22 @@ fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { ) } +fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { + PreFencingAbort { + error: truncate(error, 2048), + candidate_cleanup: candidate_cleanup_requirement( + transaction.activation.is_some(), + transaction.workload.is_some(), + ), + topology_reconciliation: transaction.abort_topology_reconciliation(), + source_cleanup: if transaction.command_kind == CommandKind::Deploy { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + }, + } +} + fn post_fencing_abort_intent( transaction: &DeploymentTransaction, error: &str, @@ -13864,7 +13881,7 @@ mod tests { } #[test] - fn a_declaring_redeploy_is_not_yielded_to_a_continuity_when_the_held_incumbent_dies() -> Result<()> { + fn a_declaring_redeploy_is_not_yielded_when_the_held_incumbent_dies() -> Result<()> { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; admit_incumbent(&world)?; @@ -13873,39 +13890,45 @@ mod tests { seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&held))?; assert_eq!(redeploy.held_disagreement(), None); workload.kill(); - for _ in 0..40 { - let _ = world.engine.run_scheduler_tick(); + for _ in 0..4 { + let _ = world.engine.supervise_one_admitted_generation()?; } let all = every_transaction(&world)?; assert!( all.iter().all(|transaction| !is_continuity_over(transaction, &held)), "a continuity was minted over the held incumbent" ); - let redeploy = all - .iter() - .find(|transaction| transaction.transaction_id == redeploy.transaction_id) - .context("the redeploy is nowhere")?; - // The fixture's one workload is the candidate too, so the redeploy may - // fail for its own reasons; it is never yielded away. + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); assert!( - redeploy - .pre_fencing_abort - .as_ref() - .is_none_or(|abort| !abort.error.contains("yielded to continuity")), + after.pre_fencing_abort.is_none(), "the redeploy yielded: {:?}", - redeploy.pre_fencing_abort + after.pre_fencing_abort ); Ok(()) } + /// The control for the test above: the same redeploy over a dead incumbent + /// that is not held is yielded to continuity. #[test] - fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { + fn a_declaring_redeploy_still_yields_to_continuity_when_the_incumbent_is_not_held() -> Result<()> { let workload = SwitchWorkload::new(); let world = EngineFixture::with_workload(workload.clone())?; let incumbent = admit_incumbent(&world)?; - // The incumbent is declared; the Deploy in front of it is the held one. - let seeded = + assert!(incumbent.readiness().is_ok()); + let redeploy = seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + workload.kill(); + world.engine.supervise_one_admitted_generation()?; + let after = resident(&world)?.value; + assert_eq!(after.transaction_id, redeploy.transaction_id); + let abort = after.pre_fencing_abort.context("the redeploy was not yielded")?; + assert!(abort.error.contains("yielded to continuity"), "{}", abort.error); + Ok(()) + } + + /// `seeded` with its own Expected undeclared, so Idunn holds it. + fn undeclared(world: &EngineFixture, seeded: &DeploymentTransaction) -> Result { let mut bare = seeded.clone(); // Undeclare the Expected and re-derive what is bound to its digest. let expected = bare.expected.as_mut().context("no expected")?; @@ -13922,21 +13945,93 @@ mod tests { bare.activation_publication_sha256 = Some(activation.canonical_sha256()?); bare.activation = Some(activation); bare.validate()?; - let stored = resident(&world)?; - replace_transaction(&world.state_store, &stored, &bare)?; assert!(bare.held_disagreement().is_some()); + Ok(bare) + } + + /// A pre-fence Deploy whose own Expected is undeclared, stored in place. + fn held_predeploy( + world: &EngineFixture, + incumbent: &AdmittedGeneration, + ) -> Result { + let seeded = + seeded_transaction(world, DeploymentPhase::Warming, CommandKind::Deploy, Some(incumbent))?; + let bare = undeclared(world, &seeded)?; + replace_transaction(&world.state_store, &resident(world)?, &bare)?; + Ok(bare) + } + + #[test] + fn a_held_predeploy_transaction_is_not_aborted_to_yield_to_continuity() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + // The incumbent is declared; the Deploy in front of it is the held one. + let held = held_predeploy(&world, &incumbent)?; workload.kill(); for _ in 0..4 { let _ = world.engine.supervise_one_admitted_generation()?; } let after = resident(&world)?.value; - assert_eq!(after.transaction_id, seeded.transaction_id); + assert_eq!(after.transaction_id, held.transaction_id); assert!(after.pre_fencing_abort.is_none(), "Idunn aborted a held record"); assert!(after.completion.is_none()); Ok(()) } + #[test] + fn cancel_withdraws_a_held_predeploy_and_leaves_the_incumbent_untouched() -> Result<()> { + let workload = SwitchWorkload::new(); + let world = EngineFixture::with_workload(workload.clone())?; + let incumbent = admit_incumbent(&world)?; + let held = held_predeploy(&world, &incumbent)?; + let before = incumbent_envelope(&world)?; + + cancel(&world.state_store, &held.command_id, "operator")?; + let cancelling = resident(&world)?.value; + assert_eq!( + cancelling.pre_fencing_abort.as_ref().map(|abort| abort.error.as_str()), + Some("cancelled by operator") + ); + assert!(cancelling.post_fencing_abort.is_none()); + + drive(&world, |transaction| transaction.completion.is_some())?; + let finished = latest(&world)?; + assert!(matches!( + finished.completion, + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); + assert!(finished.is_terminal()); + // The target is released and nothing of the incumbent was touched. + assert!(!finished.blocks_new_target_mutation()); + assert_eq!(incumbent_envelope(&world)?, before); + assert!( + every_transaction(&world)? + .iter() + .all(|transaction| !is_continuity_over(transaction, &incumbent)), + "cancelling the redeploy minted a continuity" + ); + Ok(()) + } + + #[test] + fn cancel_refuses_a_predeploy_that_is_not_held_and_a_held_one_past_the_fence() -> Result<()> { + let world = EngineFixture::with_workload(SwitchWorkload::new())?; + let incumbent = admit_incumbent(&world)?; + let seeded = + seeded_transaction(&world, DeploymentPhase::Warming, CommandKind::Deploy, Some(&incumbent))?; + assert!(seeded.held_disagreement().is_none()); + assert!(cancel(&world.state_store, &seeded.command_id, "operator").is_err()); + assert!(resident(&world)?.value.pre_fencing_abort.is_none()); + + let mut past_the_fence = undeclared(&world, &seeded)?; + past_the_fence.enter_phase(DeploymentPhase::Fencing, now_millis()?); + assert!(past_the_fence.held_disagreement().is_some()); + assert!(!cancel_is_safe_for_held_prefence(&past_the_fence)); + Ok(()) + } + // --------------------------------------------------------------------- // Route-proof readiness: a routed target that declares no Odin dependency // is admitted by Idunn's own challenges. Odin is never read. From 8432d45bbeda127b8f1423fbf262b37334e2188a Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:36:54 +0200 Subject: [PATCH 024/264] B3 follow-up: bound the chunked trailer loop, refuse signed numbers and spaced header names Trailers draw on the header budget instead of an unbounded loop. Content-Length and chunk sizes are digits only (RFC 9112), and a header name with whitespace is refused. Pins found by cargo-mutants: partial trailer, exact bounds on all three framings, one byte then stop, unterminated chunk from a peer that keeps going. ChallengeFailure::Silent documents what it actually is. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 279 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 267 insertions(+), 12 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 89d5a3a..400566b 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5566,9 +5566,10 @@ impl NginxRouteDriver { /// Why a direct presence challenge produced no document to authenticate. #[derive(Debug)] pub enum ChallengeFailure { - /// Nothing answered: the connection was refused, timed out, or dropped, or - /// the peer closed it without a byte. A process still starting does this, - /// and it is waiting, not a fault. + /// Nothing answered: the connection was refused, or no byte arrived before + /// the timeout, or the peer closed with zero bytes. A process still + /// starting does this, and it is waiting, not a fault. Once any byte has + /// arrived, a stall or a stop is `Refused`, not this. Silent(anyhow::Error), /// Something answered wrongly, or the challenge could not be made: a /// malformed or foreign answer, an oversized one, or a local configuration @@ -5725,8 +5726,9 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, // answered in full. let mut answer = Answer::new(BufReader::new(stream)); let mut head_bytes = 0_usize; - let mut head_line = |answer: &mut Answer>| { - let line = answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, "before its headers ended")?; + // Headers and chunked trailers draw on the one budget. + let mut head_line = |answer: &mut Answer>, what: &str| { + let line = answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, what)?; head_bytes += line.len() + 2; ensure_or_refuse( head_bytes <= ROUTE_HTTP_MAX_HEADER_BYTES, @@ -5734,7 +5736,8 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, )?; Ok::<_, ChallengeFailure>(line) }; - let status = head_line(&mut answer)?; + let headers = "before its headers ended"; + let status = head_line(&mut answer, headers)?; let status = std::str::from_utf8(&status) .context("stable CultNet HTTP response status is not UTF-8") .map_err(refused)?; @@ -5747,7 +5750,7 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, let mut content_length = None; let mut chunked = false; loop { - let line = head_line(&mut answer)?; + let line = head_line(&mut answer, headers)?; if line.is_empty() { break; } @@ -5758,6 +5761,12 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, .split_once(':') .context("stable CultNet HTTP response contains a malformed header") .map_err(refused)?; + // RFC 9112 5.1: a name with whitespace around it (or none) is not a + // field name, and a recipient must not guess which one was meant. + ensure_or_refuse( + !name.is_empty() && !name.bytes().any(|byte| byte.is_ascii_whitespace()), + "stable CultNet HTTP response has a malformed header name", + )?; if name.eq_ignore_ascii_case("transfer-encoding") { ensure_or_refuse( value.trim().eq_ignore_ascii_case("chunked"), @@ -5771,9 +5780,7 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, "stable CultNet HTTP response repeats Content-Length", )?; content_length = Some( - value - .trim() - .parse::() + parse_digits(value.trim_matches([' ', '\t']), 10) .context("stable CultNet HTTP Content-Length is invalid") .map_err(refused)?, ); @@ -5798,12 +5805,12 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, let size = std::str::from_utf8(&size) .ok() .and_then(|size| size.split(';').next()) - .and_then(|size| usize::from_str_radix(size.trim(), 16).ok()) + .and_then(|size| parse_digits(size.trim_matches([' ', '\t']), 16)) .context("stable CultNet HTTP chunk size is invalid") .map_err(refused)?; if size == 0 { // Trailers, if any, end at the empty line. - while !answer.read_line(ROUTE_HTTP_MAX_HEADER_BYTES, body)?.is_empty() {} + while !head_line(&mut answer, body)?.is_empty() {} return Ok(response); } ensure_or_refuse( @@ -5824,6 +5831,16 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, } } +/// A number written only in digits of `radix`: no sign, no whitespace, not +/// empty. `from_str_radix` alone accepts a leading `+`, which RFC 9112 does +/// not. +fn parse_digits(text: &str, radix: u32) -> Option { + if text.is_empty() || !text.chars().all(|digit| digit.is_digit(radix)) { + return None; + } + usize::from_str_radix(text, radix).ok() +} + /// A 2xx status line of HTTP/1.0 or HTTP/1.1. fn is_http_success(status: &str) -> bool { let Some(rest) = status @@ -8665,6 +8682,244 @@ mod tests { ); } + // --------------------------------------------------------------------- + // The reader's limits are pinned at their edges, on answers that are + // honest in every other way, so only the limit under test can refuse them + // and a peer that simply stops is not what refuses. + // --------------------------------------------------------------------- + + const CHUNKED_HEAD: &[u8] = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"; + + fn snapshot_request() -> Vec { + encode_cultnet_message_to_vec( + &CultNetMessage::SnapshotRequest { + message_id: "probe".into(), + schema_ids: None, + record_keys: None, + }, + CultNetWireContract::CultNetSchemaV0, + ) + .expect("an encodable request") + } + + /// The raw body the HTTP reader hands back, with no document judgment. + fn http_body( + reply: impl FnOnce(&str) -> Vec + Send + 'static, + ) -> Result, ChallengeFailure> { + let port = scripted_peer(reply); + request_http_snapshot(SocketAddr::from(([127, 0, 0, 1], port)), &snapshot_request()) + } + + fn tcp_body( + reply: impl FnOnce(&str) -> Vec + Send + 'static, + ) -> Result, ChallengeFailure> { + let port = framed_peer(reply); + request_tcp_snapshot(SocketAddr::from(([127, 0, 0, 1], port)), &snapshot_request()) + } + + fn assert_body_refused(outcome: Result, ChallengeFailure>, why: &str) { + assert!( + matches!(outcome, Err(ChallengeFailure::Refused(_))), + "{why} must be refused: {:?}", + outcome.map(|body| body.len()) + ); + } + + fn chunked(pieces: &[usize]) -> Vec { + let mut reply = CHUNKED_HEAD.to_vec(); + for size in pieces { + reply.extend_from_slice(format!("{size:x}\r\n").as_bytes()); + reply.extend(std::iter::repeat_n(b'a', *size)); + reply.extend_from_slice(b"\r\n"); + } + reply.extend_from_slice(b"0\r\n\r\n"); + reply + } + + #[test] + fn every_framing_takes_exactly_the_bound_and_refuses_one_byte_more() { + let most = ROUTE_SNAPSHOT_MAX_BYTES; + // Whole answers are sent in every case, so only the bound refuses. + let http_ok_of = |length: usize| move |_: &str| http_ok(&vec![b'a'; length]); + assert_eq!(http_body(http_ok_of(most)).expect("Content-Length at the bound").len(), most); + assert_body_refused(http_body(http_ok_of(most + 1)), "Content-Length over the bound"); + + for (pieces, taken) in [ + (vec![most], true), + (vec![most + 1], false), + (vec![most - 1, 1], true), + (vec![most - 1, 2], false), + ] { + let outcome = http_body(move |_| chunked(&pieces)); + if taken { + assert_eq!(outcome.expect("chunks at the bound").len(), most); + } else { + assert_body_refused(outcome, "chunks over the bound"); + } + } + + let framed = |length: usize| move |_: &str| encode_frame(&vec![b'a'; length]).expect("a frame"); + assert_eq!(tcp_body(framed(most)).expect("a frame at the bound").len(), most); + assert_eq!(tcp_body(framed(1)).expect("the smallest frame").len(), 1); + assert_body_refused(tcp_body(framed(most + 1)), "a frame over the bound"); + assert_body_refused(tcp_body(|_| 0_u32.to_be_bytes().to_vec()), "an empty frame"); + } + + #[test] + fn a_peer_that_sends_one_byte_and_stops_is_refused_not_silent() { + assert_body_refused(http_body(|_| b"H".to_vec()), "one HTTP byte"); + assert_body_refused(tcp_body(|_| vec![0]), "one TCP byte"); + assert!(matches!(http_body(|_| Vec::new()), Err(ChallengeFailure::Silent(_)))); + assert!(matches!(tcp_body(|_| Vec::new()), Err(ChallengeFailure::Silent(_)))); + } + + /// An honest chunked answer with `after_body` in place of its end. + fn chunked_honest_then(id: &str, after_body: &[u8]) -> Vec { + let body = honest_body(id); + let mut reply = CHUNKED_HEAD.to_vec(); + reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(&body); + reply.extend_from_slice(b"\r\n"); + reply.extend_from_slice(after_body); + reply + } + + #[test] + fn a_chunked_answer_must_end_its_trailers_and_terminate_each_chunk() { + // The same answer, ended properly, is taken. + challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\n\r\n")), |_| {}) + .expect("an honest chunked answer"); + challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\nU: 2\r\n\r\n")), |_| {}) + .expect("an honest chunked answer with trailers"); + + // A trailer that begins and then the peer stops. + assert_refused( + challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\n")), |_| {}), + "a partial trailer section", + ); + + // Chunk data followed by something other than CRLF, from a peer that + // then carries on as though nothing were wrong. + assert_refused( + challenge( + lingering_peer( + |id| { + let body = honest_body(id); + let mut reply = CHUNKED_HEAD.to_vec(); + reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(&body); + reply.extend_from_slice(b"XX0\r\n\r\n"); + reply + }, + LINGER, + ), + |_| {}, + ), + "a chunk that is not terminated", + ); + } + + /// Headers and trailers draw on one budget. The head is fixed, so the + /// trailer line that lands the total exactly on the bound is computed. + #[test] + fn trailers_share_the_header_budget_and_are_bounded_at_its_edge() { + // The trailer line is `T: `, counted with its CRLF, plus the + // empty line ending the trailers (2): together they fill what the head + // left of the budget. + let padding = ROUTE_HTTP_MAX_HEADER_BYTES - CHUNKED_HEAD.len() - 2 - 2 - "T: ".len(); + let reply_with = move |padding: usize| { + move |id: &str| { + let mut trailers = format!("0\r\nT: {}\r\n", "a".repeat(padding)).into_bytes(); + trailers.extend_from_slice(b"\r\n"); + chunked_honest_then(id, &trailers) + } + }; + challenge(scripted_peer(reply_with(padding)), |_| {}).expect("trailers exactly at the bound"); + + let started = Instant::now(); + assert_refused( + challenge(lingering_peer(reply_with(padding + 1), LINGER), |_| {}), + "trailers one byte over the bound", + ); + assert!(started.elapsed() < ROUTE_SNAPSHOT_TIMEOUT / 2, "the refusal waited on the socket"); + + // Trailers that never end are cut off at the bound, not at the timeout. + let started = Instant::now(); + assert_refused( + challenge( + lingering_peer( + |id| { + let line = b"T: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"; + let mut endless = b"0\r\n".to_vec(); + endless.extend_from_slice( + &line.repeat(2 * ROUTE_HTTP_MAX_HEADER_BYTES / line.len()), + ); + chunked_honest_then(id, &endless) + }, + LINGER, + ), + |_| {}, + ), + "trailers that never end", + ); + assert!(started.elapsed() < ROUTE_SNAPSHOT_TIMEOUT / 2, "the refusal waited on the socket"); + } + + /// RFC 9112 strictness, on answers that are honest but for the one field. + #[test] + fn numbers_and_header_names_are_read_strictly() { + // `{field}:{value}` then the true length, so a value of " +" is a signed one. + let content_length = |field: &'static str, value: &'static str| { + move |id: &str| { + let body = honest_body(id); + let mut reply = + format!("HTTP/1.1 200 OK\r\n{field}:{value}{}\r\n\r\n", body.len()).into_bytes(); + reply.extend_from_slice(&body); + reply + } + }; + // Optional whitespace around a value is allowed. + challenge(scripted_peer(content_length("Content-Length", " \t ")), |_| {}) + .expect("Content-Length with optional whitespace"); + for (why, field, value) in [ + ("a signed Content-Length", "Content-Length", " +"), + ("whitespace before the colon", "Content-Length ", " "), + ("a header name with leading whitespace", " Content-Length", " "), + ] { + assert_refused(challenge(scripted_peer(content_length(field, value)), |_| {}), why); + } + assert_refused( + challenge( + scripted_peer(|id| { + let body = honest_body(id); + let mut reply = format!( + "HTTP/1.1 200 OK\r\nX-Other : y\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + reply.extend_from_slice(&body); + reply + }), + |_| {}, + ), + "whitespace before the colon of another header", + ); + assert_refused( + challenge( + scripted_peer(|id| { + let body = honest_body(id); + let mut reply = CHUNKED_HEAD.to_vec(); + reply.extend_from_slice(format!("+{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(&body); + reply.extend_from_slice(b"\r\n0\r\n\r\n"); + reply + }), + |_| {}, + ), + "a signed chunk size", + ); + } + /// A reader that fails `interruptions` times with `kind`, then serves `bytes`. struct Flaky { kind: ErrorKind, From d34f5dc3f9a4fa74c2ec570c4892229ce776491f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:38:28 +0200 Subject: [PATCH 025/264] wip: show the completion in the cancel test Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index bba9e4a..26934fd 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13998,10 +13998,12 @@ mod tests { drive(&world, |transaction| transaction.completion.is_some())?; let finished = latest(&world)?; - assert!(matches!( + assert!( + matches!(finished.completion, Some(TransactionCompletion::FailedBeforeFencing { .. })), + "{:?} / {:?}", finished.completion, - Some(TransactionCompletion::FailedBeforeFencing { .. }) - )); + finished.pre_fencing_abort + ); assert!(finished.is_terminal()); // The target is released and nothing of the incumbent was touched. assert!(!finished.blocks_new_target_mutation()); From 5c49646b81e563a5242f7697704f71b3ffc7aa75 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 19:39:55 +0200 Subject: [PATCH 026/264] B3 follow-up: find the cancelled redeploy by id, not by history order Co-Authored-By: Claude Opus 5.5 --- src/control_plane.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 26934fd..7943698 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -13997,13 +13997,15 @@ mod tests { assert!(cancelling.post_fencing_abort.is_none()); drive(&world, |transaction| transaction.completion.is_some())?; - let finished = latest(&world)?; - assert!( - matches!(finished.completion, Some(TransactionCompletion::FailedBeforeFencing { .. })), - "{:?} / {:?}", + // History also holds the incumbent's own admission: find ours by id. + let finished = every_transaction(&world)? + .into_iter() + .find(|transaction| transaction.transaction_id == held.transaction_id) + .context("the cancelled redeploy is nowhere")?; + assert!(matches!( finished.completion, - finished.pre_fencing_abort - ); + Some(TransactionCompletion::FailedBeforeFencing { .. }) + )); assert!(finished.is_terminal()); // The target is released and nothing of the incumbent was touched. assert!(!finished.blocks_new_target_mutation()); From ae3788909ad745aa4ad96483abc507beaacba3eb Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:15:08 +0200 Subject: [PATCH 027/264] Map: B3 follow-ups and the dead-hold report ruling Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 54502d5..241115c 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -73,6 +73,23 @@ live host on install. 294 passed, 2 ignored. `cargo mutants --in-diff`: 53 caugh record on its own. An operator can discard a held Deploy that has not reached the fence with `idunn cancel`. Nothing is fenced, so the incumbent is untouched. +**B3 follow-ups landed on `idunn/route-b3` at `5c49646`, 2026-09-30.** +- Cancel accepts a held pre-fence Deploy. It shares `pre_fencing_abort_intent` with the scheduler. +- The P1b test is now in direct-supervision form, with a positive control. +- The reader is strict: trailers share the 32 KiB header budget; digits only; header names without whitespace. +- The `Silent` doc is corrected. +- Tests: 301 pass. `a_failing_step_is_retried_after_a_backoff_not_every_tick` is flaky under a full parallel + run, on the base too. +- `cargo mutants --in-diff`: 24 caught, 0 missed. +- A narrow Soul pass gates the install. + +**Ruling, 2026-09-30 (operator): the dead-hold report is stored on the generation.** +- Add an optional `last_error` to `AdmittedGeneration` (new key 21), and make `status` render admitted + generations. +- It is cleared when the generation is replaced or its hold is released. +- The `generation:` readiness report gets the same home. +- It is queued after the ship, with the B2 rework. + **Odin diagnosis (Eyes, read-only, 2026-09-30).** - The 109 Odin route timeouts are self-inflicted. Live Idunn (`8ae00a1`, pre-S1) reloads nginx inside `restore_admitted_membership` before every Odin challenge. The reload moves the reuseport UDP flow, so From d3db58227a56361811f3a78933e575de3a826ebf Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 20:57:45 +0200 Subject: [PATCH 028/264] Define the frozen-source symlink limit on Windows so idunn-host builds The constant was unix-only since 9475dfa while its uses are not, so the Windows idunn-host has not compiled since S1. Found installing B3: Raven's actuator (built 2026-09-11) cannot decode the new plan types. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 400566b..fbf9558 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -68,6 +68,9 @@ struct GitTreeEntry { /// finally closed it. #[cfg(unix)] const FROZEN_SOURCE_SYMLINK_TARGET_LIMIT: usize = libc::PATH_MAX as usize; +/// Windows' long-path limit, in UTF-16 units; the same bound on a symlink blob's size. +#[cfg(not(unix))] +const FROZEN_SOURCE_SYMLINK_TARGET_LIMIT: usize = 32_767; /// F2/F3 layer (b): the only way `materialize_tree_raw` is allowed to obtain /// a directory to write into. `path` must already be a directory this From 28662cb807c3dd7cb2af5b66b64a3e3c95c6617f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:05:35 +0200 Subject: [PATCH 029/264] Map: ship log for the B3 install, the Raven actuator and the bulk fetch bug Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 241115c..49696b1 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -83,6 +83,22 @@ Nothing is fenced, so the incumbent is untouched. - `cargo mutants --in-diff`: 24 caught, 0 missed. - A narrow Soul pass gates the install. +**Ship log, 2026-09-29 (host clock).** +- B3 merged at `d32395a` and installed on Yggdrasil at 18:36 UTC, with sha `854aa351`. It migrated 5 control + records. The store backup is `/root/idunn-store-backup-20260929T183604Z-pre-route-b3`. +- Nginx reloads fell from 82 per 10 minutes to 0. Odin continuity timeouts fell from 92 in 6 hours to 0. +- **Raven's actuator (built 2026-09-11) could not decode the new plan types.** A rebuild then found that the + Windows `idunn-host` has not compiled since S1: `FROZEN_SOURCE_SYMLINK_TARGET_LIMIT` was defined for unix + only. That is fixed at `d3db582`. + - The new actuator (sha `f36bf1fc`) is swapped in on Raven. The old one is kept as + `idunn-host.exe.prev-20260929T190006Z`. + - Follow-up: nothing builds the Windows actuator in verification. +- raven-muninn was admitted at 19:02 by Odin correlation, replacing the held generation. +- **The streampixels-service deploy failed before fencing.** The S1 `bulk_fetch_objects` asks for bare blob + ids with `git fetch --stdin`. In a `blob:none` clone, the connectivity check then fails with "bad revision + ". The fix, with a test, is in Hands on `fix/bulk-blob-fetch`: a promisor-style fetch with + `--filter=blob:none` and `fetch.negotiationAlgorithm=noop`. + **Ruling, 2026-09-30 (operator): the dead-hold report is stored on the generation.** - Add an optional `last_error` to `AdmittedGeneration` (new key 21), and make `status` render admitted generations. From 7f776ec3a6fe1c384e304f539037de1d187faa5c Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:06:03 +0200 Subject: [PATCH 030/264] Pin bulk_fetch_objects against a real blob:none partial clone Every existing source-driver fixture uses an origin without uploadpack.allowFilter, so --filter=blob:none is ignored, the clone is full, and bulk_fetch_objects never has a missing blob to fetch. This fixture advertises the filter, so the checkout really lacks the blob, and checks delivery with GIT_NO_LAZY_FETCH=1 so only the bulk fetch can supply it. It fails on the current fetch arguments. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index fbf9558..83a76c6 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -10557,6 +10557,72 @@ Content-Le".to_vec()), |_| {}), Ok(()) } + /// A real partial clone: the origin advertises `uploadpack.allowFilter`, + /// so `--filter=blob:none` is honoured and the checkout holds no blobs. + /// Every other fixture's origin ignores the filter and clones in full, + /// so `bulk_fetch_objects` never had a missing blob to fetch and never + /// met Git's post-fetch connectivity check on a bare blob id. The blob + /// is checked with `GIT_NO_LAZY_FETCH=1`, so only the bulk fetch itself + /// can have delivered it. + #[cfg(unix)] + #[test] + fn bulk_fetch_objects_delivers_a_blob_into_a_blob_none_partial_clone() -> Result<()> { + let temp = tempfile::tempdir()?; + let origin_repo = temp.path().join("origin"); + fs::create_dir(&origin_repo)?; + git_at(&origin_repo, &["init", "--initial-branch=main"])?; + git_at(&origin_repo, &["config", "user.name", "Idunn Test"])?; + git_at( + &origin_repo, + &["config", "user.email", "idunn-test@example.invalid"], + )?; + git_at(&origin_repo, &["config", "uploadpack.allowFilter", "true"])?; + git_at(&origin_repo, &["config", "uploadpack.allowAnySHA1InWant", "true"])?; + fs::write(origin_repo.join("payload.txt"), b"partial clone payload\n")?; + git_at(&origin_repo, &["add", "--all"])?; + git_at(&origin_repo, &["commit", "-m", "fixture"])?; + let blob = git_at(&origin_repo, &["rev-parse", "HEAD:payload.txt"])?; + + let source_cache_root = temp.path().join("source-cache"); + fs::create_dir(&source_cache_root)?; + let driver = GitSourceDriver::new( + source_cache_root.clone(), + temp.path().join("frozen"), + None, + ); + let checkout = source_cache_root.join("checkout"); + driver.git([ + OsString::from("clone"), + OsString::from("--filter=blob:none"), + OsString::from("--no-checkout"), + OsString::from(format!("file://{}", origin_repo.display())), + checkout.as_os_str().to_owned(), + ])?; + + let blob_present = || -> Result { + Ok(Command::new("/usr/bin/git") + .arg("-C") + .arg(&checkout) + .args(["cat-file", "-t", &blob]) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("GIT_NO_LAZY_FETCH", "1") + .output()? + .status + .success()) + }; + ensure!( + !blob_present()?, + "fixture is not a partial clone: the blob is already local" + ); + driver.bulk_fetch_objects(&checkout, std::slice::from_ref(&blob))?; + assert!( + blob_present()?, + "bulk fetch succeeded but did not deliver the blob" + ); + Ok(()) + } + /// F2/F3 (Self's rulings, second Cut 1 fix batch): a tree with two /// entries literally named `a` at the same level -- one a symlink /// pointing outside the frozen root, one a subtree holding `pwn` -- must From f9cebb33d0ce12514c31ff7b0eb45819d66937e2 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:12:06 +0200 Subject: [PATCH 031/264] Pin the partial-clone bulk fetch arguments in the spy test A file:// origin delivers an explicitly wanted blob with or without the promisor-style flags, so the real-shape fixture cannot fail on the bug that GitHub triggers. The spy test already records the bulk fetch line; assert it carries fetch.negotiationAlgorithm=noop, --recurse-submodules=no and --filter=blob:none. Fails on the current arguments. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index 83a76c6..c39f0bd 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -10554,6 +10554,24 @@ Content-Le".to_vec()), |_| {}), bulk_fetches, 1, "expected exactly one bulk --stdin fetch; invocations:\n{log_text}" ); + // Git's own lazy fetch for a promisor remote sends exactly these; a + // hosted origin (GitHub) omits an explicitly wanted blob, and Git's + // connectivity check then fails, without them. A file:// origin + // delivers the blob either way, so only the arguments can pin this. + let bulk_line = log_text + .lines() + .find(|line| line.contains("fetch") && line.contains("--stdin")) + .unwrap_or_default(); + for required in [ + "fetch.negotiationAlgorithm=noop", + "--recurse-submodules=no", + "--filter=blob:none", + ] { + assert!( + bulk_line.contains(required), + "bulk fetch lacks {required}: {bulk_line}" + ); + } Ok(()) } From 08d94ee61be50cab10b70a8590b3fa8308c42141 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:13:22 +0200 Subject: [PATCH 032/264] Fetch blobs in bulk the way Git's lazy fetch does for a promisor remote In a blob:none partial clone, fetching bare blob ids with --stdin made Git's post-fetch connectivity check treat the id as a revision and fail with "bad revision" / "did not send all necessary objects" against GitHub, which broke a live deploy fetching a vendor gitlink's blob. Add fetch.negotiationAlgorithm=noop, --recurse-submodules=no and --filter=blob:none, the arguments Git's own lazy fetch uses. Co-Authored-By: Claude Opus 5.5 --- src/drivers.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/drivers.rs b/src/drivers.rs index c39f0bd..6af3559 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -1133,6 +1133,10 @@ impl GitSourceDriver { /// in depth: Git itself refuses a fetched tree with duplicate names /// before any of it reaches disk, independent of the explicit `fsck` /// call S1 added around the revision this bulk fetch serves. + /// The noop negotiation, `--recurse-submodules=no` and explicit + /// `--filter=blob:none` are the arguments Git's own lazy fetch uses for a + /// promisor remote; without them a hosted origin omits the wanted blob and + /// Git's connectivity check fails on the bare blob id. fn bulk_fetch_objects(&self, repository: &Path, objects: &[String]) -> Result<()> { if objects.is_empty() { return Ok(()); @@ -1155,11 +1159,15 @@ impl GitSourceDriver { let mut command = self.git_command([ OsString::from("-c"), OsString::from("transfer.fsckObjects=true"), + OsString::from("-c"), + OsString::from("fetch.negotiationAlgorithm=noop"), OsString::from("-C"), repository.as_os_str().to_owned(), OsString::from("fetch"), OsString::from("--no-tags"), OsString::from("--no-write-fetch-head"), + OsString::from("--recurse-submodules=no"), + OsString::from("--filter=blob:none"), OsString::from("--stdin"), OsString::from("origin"), ])?; From 7480ee9d1c7288a27ed63859ee7d62bb9dab28d1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 21:55:34 +0200 Subject: [PATCH 033/264] Map: StreamPixels ship completed through Idunn Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 49696b1..caff977 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -99,6 +99,33 @@ Nothing is fenced, so the incumbent is untouched. ". The fix, with a test, is in Hands on `fix/bulk-blob-fetch`: a promisor-style fetch with `--filter=blob:none` and `fetch.negotiationAlgorithm=noop`. +**Ship completed, 2026-09-29 (host clock).** +- **Bulk-fetch fix.** Merged at `583b2a7` and installed at 19:17. The Idunn sha is `01bde676`. +- **The first streampixels-service redeploy failed after fencing.** The incumbent was restored, and every restart + was refused by Odin with "runtime presence publisher sequence was reordered". Cause: Odin compared publisher + sequences across incarnations, and the TS publisher counts from 1 per process. + - The operator ruled: fix Odin's rule. + - Odin `db041ec` compares sequences only while the stored presence still authenticates under the current + activation. Soul said deploy, with follow-ups; Odin was deployed at 19:4x. +- **Admissions.** + - streampixels-service was admitted at 19:50 on s3-c2. + - The web binding dropped `STREAMPIXELS_ODIN_CULTMESH_RUDP`, and gamecult-ops `37b5ed9` does the same. + - streampixels-web was admitted with Odin-free readiness. +- **Public cutover at 19:54.** The vhost upstreams moved to `:8833` (web) and `:8832` (service). The backup is + `/root/streampixels.gamecult.org.conf.bak-20260929-195415-pre-idunn-upstreams`. + - `https://streampixels.gamecult.org/api/catalog` returns 200 (120,893 B), after being 502 since migration 013. + - The overlay SSE stream answers `text/event-stream`, and the ops checker passes. +- **Legacy units.** `streampixels-web.service` is disabled and stopped. `streampixels-service.service` is + disabled, and was already stopped. +- **Follow-ups.** + - The C2 acceptance hour: sample the nginx reload counter at T0 and T0+60, with no rejected challenges. + - Nothing builds the Windows `idunn-host` in verification. + - Odin presence: pin the stored-at re-authentication time (Soul's M3 survived), and consider checking activation + identity directly rather than any failure to authenticate. + - The dead-hold report field (ruled). + - B2 rework on B3. + - Odin's CultLib pin bump, after the stray-packet merge. + **Ruling, 2026-09-30 (operator): the dead-hold report is stored on the generation.** - Add an optional `last_error` to `AdmittedGeneration` (new key 21), and make `status` render admitted generations. From 94c7691cb610aea8b43f00bedae1ea7178fa5599 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 22:56:10 +0200 Subject: [PATCH 034/264] Map: the C2 acceptance hour passed; Odin 5c37860 deployed Co-Authored-By: Claude Opus 5.5 --- docs/route-continuity-cut.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index caff977..f8c8306 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -126,6 +126,16 @@ Nothing is fenced, so the incumbent is untouched. - B2 rework on B3. - Odin's CultLib pin bump, after the stray-packet merge. +**C2 acceptance hour passed, 2026-09-29 19:55-20:55 UTC (21:55-22:55 CEST).** +- 1 nginx reload, which was Odin's own route promotion during its `5c37860` deploy. +- 0 rejected StreamPixels challenges, 0 Odin continuity timeouts, and Idunn `NRestarts=0`. +- One running unit each for service, web and Odin. Public `/` and `/api/catalog` return 200. +- Also deployed in the window: Odin `5c37860`, carrying CultLib `3bf1c0c` (RUDP stray-packet hardening in every + runtime), Sleipnir removed, and presence ordering decided by activation identity. +- Still open for Odin: it exits when its 64-slot RUDP session table fills, because its own heartbeat publish + fails. Publishers that never Disconnect hold slots for 30 s. The fix is in Hands. +- The second run, with Odin broken on purpose, remains the operator's to schedule. + **Ruling, 2026-09-30 (operator): the dead-hold report is stored on the generation.** - Add an optional `last_error` to `AdmittedGeneration` (new key 21), and make `status` render admitted generations. From 5f8417589d22f84a306d0329444a41d01884eb4f Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:38:26 +0200 Subject: [PATCH 035/264] Candidate cleanup owed is one method both abort intents share The two-flag helper let either call site swap or fold an argument unnoticed. DeploymentTransaction::candidate_cleanup_owed reads the transaction itself, and a four-row table drives the method and both intents. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 77 ++++++++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 7943698..c068360 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -1370,6 +1370,17 @@ impl DeploymentTransaction { } } + /// Whether an abort owes the candidate's durable cleanup: it does once an + /// activation was issued or a workload was observed, either of which leaves + /// something behind. + fn candidate_cleanup_owed(&self) -> CleanupEvidence { + if self.activation.is_some() || self.workload.is_some() { + CleanupEvidence::Pending + } else { + CleanupEvidence::Skipped + } + } + fn rejected(command: &DeploymentCommand, error: anyhow::Error, now: u64) -> Result { let mut transaction = Self::new(command, command.selector.clone(), 0, None, now)?; let detail = truncate(&format!("{error:#}"), 2048); @@ -7991,10 +8002,7 @@ fn incumbent_was_stopped_during_fencing(fencing: &FencingEvidence) -> bool { fn pre_fencing_abort_intent(transaction: &DeploymentTransaction, error: &str) -> PreFencingAbort { PreFencingAbort { error: truncate(error, 2048), - candidate_cleanup: candidate_cleanup_requirement( - transaction.activation.is_some(), - transaction.workload.is_some(), - ), + candidate_cleanup: transaction.candidate_cleanup_owed(), topology_reconciliation: transaction.abort_topology_reconciliation(), source_cleanup: if transaction.command_kind == CommandKind::Deploy { CleanupEvidence::Pending @@ -8025,10 +8033,7 @@ fn post_fencing_abort_intent( } else { CleanupEvidence::Skipped }, - candidate_cleanup: candidate_cleanup_requirement( - transaction.activation.is_some(), - transaction.workload.is_some(), - ), + candidate_cleanup: transaction.candidate_cleanup_owed(), topology_reconciliation: transaction.abort_topology_reconciliation(), source_cleanup: if transaction.command_kind == CommandKind::Deploy { CleanupEvidence::Pending @@ -8038,17 +8043,6 @@ fn post_fencing_abort_intent( } } -fn candidate_cleanup_requirement( - has_prepared_activation: bool, - has_workload_observation: bool, -) -> CleanupEvidence { - if has_prepared_activation || has_workload_observation { - CleanupEvidence::Pending - } else { - CleanupEvidence::Skipped - } -} - fn sequence_requires_admission( latest: Option<&TopologyEvidence>, maximum_admitted_sequence: u64, @@ -9803,19 +9797,38 @@ mod tests { } #[test] - fn activation_without_workload_still_requires_durable_candidate_cleanup() { - assert_eq!( - candidate_cleanup_requirement(true, false), - CleanupEvidence::Pending - ); - assert_eq!( - candidate_cleanup_requirement(true, true), - CleanupEvidence::Pending - ); - assert_eq!( - candidate_cleanup_requirement(false, false), - CleanupEvidence::Skipped - ); + fn candidate_cleanup_is_owed_when_an_activation_or_a_workload_exists() -> Result<()> { + // (activation issued, workload observed, expected cleanup) + let rows = [ + (false, false, CleanupEvidence::Skipped), + (true, false, CleanupEvidence::Pending), + (false, true, CleanupEvidence::Pending), + (true, true, CleanupEvidence::Pending), + ]; + for (has_activation, has_workload, expected) in rows { + let world = ContinuityProjection::after_candidate_observed()?; + let mut transaction = world.continuity_transaction()?; + if !has_activation { + transaction.activation = None; + } + transaction.workload = has_workload.then(|| workload(1000, 1, 2)); + assert_eq!( + transaction.candidate_cleanup_owed(), + expected, + "activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + pre_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "pre-fencing intent, activation {has_activation}, workload {has_workload}" + ); + assert_eq!( + post_fencing_abort_intent(&transaction, "x").candidate_cleanup, + expected, + "post-fencing intent, activation {has_activation}, workload {has_workload}" + ); + } + Ok(()) } #[test] From bf5347c3704d2f7bb5626c5c284a4d97ba0eeb2b Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:40:11 +0200 Subject: [PATCH 036/264] The chunk-size line admits whitespace only before an extension RFC 9112 7.1 allows whitespace before a chunk extension and nowhere else. The reader trimmed both ends of the size, so " 3" and "3 " were taken. It now splits at the first ';', trims trailing SP and HTAB only when an extension follows, and leaves parse_digits as the digit authority. Rows use an honest body so only the size line differs. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 6af3559..dd9f6c0 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -5813,10 +5813,16 @@ fn request_http_snapshot(target: SocketAddr, payload: &[u8]) -> Result, let mut response = Vec::new(); loop { let size = answer.read_line(ROUTE_HTTP_CHUNK_LINE_BYTES, body)?; + // RFC 9112 7.1: `chunk-size [ chunk-ext ] CRLF`, where an + // extension may follow whitespace. Nothing else is trimmed. let size = std::str::from_utf8(&size) .ok() - .and_then(|size| size.split(';').next()) - .and_then(|size| parse_digits(size.trim_matches([' ', '\t']), 16)) + .and_then(|line| match line.split_once(';') { + Some((digits, _extension)) => { + parse_digits(digits.trim_end_matches([' ', '\t']), 16) + } + None => parse_digits(line, 16), + }) .context("stable CultNet HTTP chunk size is invalid") .map_err(refused)?; if size == 0 { @@ -8786,9 +8792,15 @@ mod tests { /// An honest chunked answer with `after_body` in place of its end. fn chunked_honest_then(id: &str, after_body: &[u8]) -> Vec { + chunked_honest_sized(id, "", "", after_body) + } + + /// An honest chunked answer whose chunk-size line is the hex length between + /// `before` and `after`. + fn chunked_honest_sized(id: &str, before: &str, after: &str, after_body: &[u8]) -> Vec { let body = honest_body(id); let mut reply = CHUNKED_HEAD.to_vec(); - reply.extend_from_slice(format!("{:x}\r\n", body.len()).as_bytes()); + reply.extend_from_slice(format!("{before}{:x}{after}\r\n", body.len()).as_bytes()); reply.extend_from_slice(&body); reply.extend_from_slice(b"\r\n"); reply.extend_from_slice(after_body); @@ -8803,6 +8815,30 @@ mod tests { challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\nU: 2\r\n\r\n")), |_| {}) .expect("an honest chunked answer with trailers"); + // RFC 9112 7.1: whitespace may precede a chunk extension and nothing + // else. The answer is otherwise honest, so only the size line differs. + for (before, after) in [("", ";x"), ("", " ;x"), ("", "\t;x"), ("", ";x=1;y")] { + challenge( + scripted_peer(move |id| chunked_honest_sized(id, before, after, b"0\r\n\r\n")), + |_| {}, + ) + .unwrap_or_else(|error| panic!("size line {before:?}+size+{after:?}: {error:?}")); + } + for (why, before, after) in [ + ("whitespace before the chunk size", " ", ""), + ("a tab before the chunk size", "\t", ""), + ("whitespace after the chunk size and no extension", "", " "), + ("whitespace before the size, ahead of an extension", " ", ";x"), + ] { + assert_refused( + challenge( + scripted_peer(move |id| chunked_honest_sized(id, before, after, b"0\r\n\r\n")), + |_| {}, + ), + why, + ); + } + // A trailer that begins and then the peer stops. assert_refused( challenge(scripted_peer(|id| chunked_honest_then(id, b"0\r\nT: 1\r\n")), |_| {}), From 97fb2b88719077d279d1b55618672441e677dcf1 Mon Sep 17 00:00:00 2001 From: Meta Vens Date: Tue, 29 Sep 2026 23:44:25 +0200 Subject: [PATCH 037/264] Map: B2 rework plan, D1/T1/T2/W1/R1 cuts, and the 2026-09-30 rulings Splice the B2 rework map (owner map, per-file cut, verification, live relevance, subtraction estimate) under B2; add cuts D1, T1, T2, W1 and R1; record Q-B2-1 (a), Q-B2-2 (fail at once), Q-R1 (a) and Q-B4 (routed by Idunn's route proof, unrouted by admission plus workload; the Odin non-Ready veto clause held open). B4 is blocked on the Odin-lifecycle authority audit. B2, D1, R1 ready for Hands; W1, T1, T2 in progress on hands/idunn-w1-t1-t2. Co-Authored-By: Claude Sonnet 5.5 --- docs/route-continuity-cut.md | 745 ++++++++++++++++++++++++++++++++++- 1 file changed, 729 insertions(+), 16 deletions(-) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index f8c8306..bba4d5f 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -193,7 +193,7 @@ it for merge. - B2 also does not compile on top of B3, and after a one-line fix two tests fail, because the fixture target is now route-proof. -The rework runs after B3 merges and is rebased onto it. +The rework is a fresh branch from `583b2a7`, not a rebase of `1786ddf`. Its map is under B2 in section 4. **Rulings, 2026-09-30.** - **The ceiling never refuses a rollback.** Restoring the admitted or incumbent route is survival, not @@ -203,6 +203,27 @@ The rework runs after B3 merges and is rebased onto it. admitted generation, so a target with no admitted generation is charged too. That schema change is folded into the store-version bump that F1 needs: a v4 generation, with a typed lift from v3. +**Rulings, 2026-09-30 (operator, on the B2 rework map).** +- **Q-B2-1 (a):** the continuity restart log moves into `TargetSupervision`; the restart ceiling reads the + target's own log; the carry compensator is deleted. +- **Q-B2-2: fail at once.** A deploy refused by the route/actuation ceiling before the fence fails immediately, + with the reopen time in the error. +- **Q-R1 (a):** the route declaration is the challenge declaration. Every routed target answers the + stable-route challenge regardless of readiness class. R1 checks it before the fence; H1 gives Heimdall a + responder. Operator note: the question's first framing was confusing; the cause is that Heimdall has no + responder, not its class. +- **Q-B4: Idunn's own proof.** Routed providers by current route proof and not degraded; unrouted providers by + admission plus a live workload observation. One clause is **held open**: "a current Odin non-Ready word + still excludes a provider" is not ruled, pending the Odin-lifecycle authority audit (a separate Imagination + pass). Operator: "the last few rulings on Odin are making my authority sense tingle, it feels like Odin is + doing lifecycle work when that's very much Idunn's wheelhouse." B4 is blocked on that audit. + +**Cut status, 2026-09-30.** +- **Ready for Hands:** B2 (rework map in section 4); D1 (after B2); R1 (after B2). +- **In progress** on `hands/idunn-w1-t1-t2`: W1, T1, T2. +- **Blocked:** B4, on the Odin-lifecycle authority audit; H1 (Heimdall repo), on CultLib C1. + + Heads read for this map: | Repo | Ref | Notes | @@ -571,6 +592,97 @@ it. This is a CultLib foundation merge, so the operator should rule on it. Depends on it: C1 and the StreamPixels gitlink in C2. +**Q-B2-1. Does the continuity restart log move into `TargetSupervision` too?** + +**RULED (a) by the operator, 2026-09-30.** + +Options: +- (a) Move it. `ContinuityBackoff`'s own doc (`:907-908`) already says it "belongs to the target, not the + generation". On the generation it survives only through the copy in `from_transaction` (`:2074-2076`). + Moving it deletes that carry and key 20. The mint CAS pins the target record instead of racing the + generation, which is the F3 shape. One record holds both of a target's meters. +- (b) Keep it on the generation, reshaped to the sliding log in place. The route meter still needs + `TargetSupervision` for first deploys, so the meters split across two records and the carry stays. + +Outcome: the continuity restart log lives in `TargetSupervision`; the restart ceiling reads the target's own +log; the carry compensator in `from_transaction` is deleted. + +Depends on it: B2 (see the B2 rework map in section 4). + +**Q-B2-2. When the route ceiling refuses a first or forward deploy before the fence, does `idunn up` fail or +wait?** + +**RULED: fail at once, by the operator, 2026-09-30.** + +Options: +- (a) Fail. The deploy is aborted pre-fence with "route actuation ceiling reached for ; reopens at +