From a484a7ec17befbe6a5454ebf31f5ac2289b46636 Mon Sep 17 00:00:00 2001 From: Eureka Date: Fri, 13 Nov 2026 18:57:59 +0000 Subject: [PATCH 01/10] Say why the stable route proof failed: journal each distinct reason, keep it in the deadline abort advance_routing now logs the proof failure (silent, refused with the HTTP status line, presence not active, authority disagreement) once per distinct reason, and the Routing deadline abort carries the step's last failure instead of only the clock. No persisted type or schema changes. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 107 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 34d1b87..d680d27 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -6730,10 +6730,23 @@ impl Engine { } fn clear_fault(&self, transaction_id: &str) { - self.fault_reports - .lock() - .expect("fault report mutex") - .remove(transaction_id); + let mut reports = self.fault_reports.lock().expect("fault report mutex"); + reports.remove(transaction_id); + reports.remove(&route_proof_report_key(transaction_id)); + } + + /// Say why the stable route's proof failed. The Routing phase retries the + /// proof until its deadline, and the deadline's own cause names only the + /// clock, so this line is where an operator learns what the listener + /// actually did. Said once per distinct reason while the fault lasts. + fn report_route_proof_failure(&self, transaction_id: &str, error: &anyhow::Error) { + self.report_once( + &route_proof_report_key(transaction_id), + format!( + "Idunn route proof for {transaction_id} failed: {}", + route_proof_reason(error) + ), + ); } /// Write `detail` as the record's `last_error`, touching nothing else and @@ -8871,11 +8884,13 @@ impl Engine { ) { Ok(observation) => observation, Err(proof_error) if !rollback_allowed => { + self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return Err(proof_error).context( "incumbent authority was fenced; candidate route remains installed for fail-closed retry", ); } Err(proof_error) => { + self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return match driver.rollback( expected, &activation.runtime_instance_id, @@ -10560,8 +10575,15 @@ impl Engine { .iter() .any(|request| request.value.command_id == transaction.command_id); let cause = if now >= deadline.deadline_at_unix_millis { + // The step's latest failure is the reason the phase never + // finished, and the abort that ends it must not drop it. + let last_attempt = transaction + .last_error + .as_deref() + .map(|error| format!("; its last attempt failed: {error}")) + .unwrap_or_default(); format!( - "the {:?} phase passed its deadline at unix ms {}", + "the {:?} phase passed its deadline at unix ms {}{last_attempt}", transaction.phase, deadline.deadline_at_unix_millis ) } else if requested { @@ -11426,6 +11448,22 @@ fn runner_failure_journal_line( } } +fn route_proof_report_key(transaction_id: &str) -> String { + format!("route-proof:{transaction_id}") +} + +/// The stable route proof's failure in words: whether the listener said +/// nothing or answered wrongly, then the specific refusal (HTTP status line, +/// presence not active, challenge or authority disagreement). The texts it +/// carries name states and codes, never keys or signature bytes. +fn route_proof_reason(error: &anyhow::Error) -> String { + match error.downcast_ref::() { + Some(ChallengeFailure::Silent(_)) => format!("silent, nothing answered: {error:#}"), + Some(ChallengeFailure::Refused(_)) => format!("refused: {error:#}"), + None => format!("{error:#}"), + } +} + fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { @@ -16954,6 +16992,60 @@ mod tests { Ok(()) } + #[test] + fn a_route_proof_failure_names_what_the_listener_did() { + let reason = |error: anyhow::Error| route_proof_reason(&error); + let silent = anyhow::Error::new(ChallengeFailure::Silent(anyhow!( + "connecting stable CultNet HTTP route 127.0.0.1:9: refused" + ))); + let text = reason(silent); + assert!(text.contains("silent") && text.contains("127.0.0.1:9"), "{text}"); + + let status = anyhow::Error::new(ChallengeFailure::Refused(anyhow!( + "stable CultNet HTTP route rejected the snapshot challenge: HTTP/1.1 502 Bad Gateway" + ))) + .context("candidate did not answer its stable route challenge"); + let text = reason(status); + assert!(text.contains("refused") && text.contains("502 Bad Gateway"), "{text}"); + + let inactive = anyhow!("route proof runtime is warming, not active"); + assert!(reason(inactive).contains("warming, not active")); + + let disagrees = anyhow::Error::new(PresenceDisagrees { + disagreements: vec![OdinTopologyDisagreement { + code: "runtime-instance-mismatch".into(), + expected: Some("a".into()), + observed: Some("b".into()), + }], + }); + let text = reason(disagrees); + assert!(text.contains("runtime-instance-mismatch (expected a, observed b)"), "{text}"); + + let unbound = anyhow!("route proof response is not bound to the exact challenge"); + assert!(reason(unbound).contains("not bound to the exact challenge")); + } + + #[test] + fn a_route_proof_failure_is_reported_once_per_reason_and_cleared_on_recovery() -> Result<()> { + let world = EngineFixture::new()?; + let key = route_proof_report_key("tx-1"); + let said = |world: &EngineFixture| { + world + .engine + .fault_reports + .lock() + .unwrap() + .get(&key) + .and_then(|report| report.last.lock().unwrap().clone()) + }; + world.engine.report_route_proof_failure("tx-1", &anyhow!("route proof runtime is warming, not active")); + let first = said(&world).context("no report")?; + assert!(first.contains("tx-1") && first.contains("warming, not active"), "{first}"); + world.engine.clear_fault("tx-1"); + assert!(said(&world).is_none(), "recovery left the proof report armed"); + Ok(()) + } + #[test] fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { write_fixture_anchor()?; @@ -24910,6 +25002,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .post_fencing_abort .context("the passed deadline did not end a deployment")?; assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!( + abort.error.contains("its last attempt failed"), + "the deadline abort dropped the proof's failure: {}", + abort.error + ); assert!( routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), "the boot lift took a deployment's abort" From 88e2f24cc67888b432daa3e55b99c50078ff3b57 Mon Sep 17 00:00:00 2001 From: Eureka Date: Sat, 14 Nov 2026 01:18:38 +0000 Subject: [PATCH 02/10] Test the route-proof report through advance_routing: reported once per reason, cleared on recovery; deadline abort carries the proof's own failure text Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index d680d27..11db393 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -24349,9 +24349,39 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.run_to_routing()?; let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; routed.stub.hang_up.store(true, Ordering::SeqCst); - for _ in 0..3 { + // The Routing tick itself reports why the proof failed, once per + // distinct reason, and recovery clears it. + let proof_key = route_proof_report_key(&routed.transaction()?.transaction_id); + let said = |routed: &RoutedWorld| -> Option { + routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .get(&proof_key) + .and_then(|report| report.last.lock().unwrap().clone()) + }; + assert!(said(&routed).is_none(), "a proof report existed before any failure"); + routed.resume()?; + let first = said(&routed).context("the failing Routing tick reported no proof failure")?; + assert!(first.contains("route proof") && first.contains("silent, nothing answered"), "{first}"); + // The hung-up stub fails the same way each attempt, but the OS + // words the tail either as an early EOF or a broken pipe, so a + // retry may honestly report a second distinct reason. What may + // not change is everything up to that tail, and a repeat must not + // mint a new line each attempt. + let head = |text: &str| text.split("answered:").next().unwrap().to_owned(); + let mut seen = vec![first.clone()]; + for _ in 0..2 { routed.resume()?; + let again = said(&routed).context("a retry cleared the proof report")?; + assert_eq!(head(&again), head(&first), "a retry reworded the report"); + if !seen.contains(&again) { + seen.push(again); + } } + assert!(seen.len() <= 2, "the same failure was reported afresh on every retry: {seen:#?}"); // Before the deadline each attempt installs and rolls back, and no // resume moves the deadline. assert_eq!(routed.reloads(), 6); @@ -24394,6 +24424,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(records.len(), 1); assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + assert!(said(&routed).is_none(), "recovery left the proof report armed"); Ok(()) } @@ -25007,6 +25038,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide "the deadline abort dropped the proof's failure: {}", abort.error ); + assert!( + abort.error.contains("stable CultNet route stopped before its headers ended"), + "the deadline abort dropped the proof's own failure text: {}", + abort.error + ); assert!( routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), "the boot lift took a deployment's abort" From 7eae1c4ee131e623e1f53118eb307d36b49921d2 Mon Sep 17 00:00:00 2001 From: Eureka Date: Sat, 14 Nov 2026 01:34:34 +0000 Subject: [PATCH 03/10] Say why the stable route proof failed (#6) --- src/control_plane.rs | 145 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 139 insertions(+), 6 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 34d1b87..11db393 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -6730,10 +6730,23 @@ impl Engine { } fn clear_fault(&self, transaction_id: &str) { - self.fault_reports - .lock() - .expect("fault report mutex") - .remove(transaction_id); + let mut reports = self.fault_reports.lock().expect("fault report mutex"); + reports.remove(transaction_id); + reports.remove(&route_proof_report_key(transaction_id)); + } + + /// Say why the stable route's proof failed. The Routing phase retries the + /// proof until its deadline, and the deadline's own cause names only the + /// clock, so this line is where an operator learns what the listener + /// actually did. Said once per distinct reason while the fault lasts. + fn report_route_proof_failure(&self, transaction_id: &str, error: &anyhow::Error) { + self.report_once( + &route_proof_report_key(transaction_id), + format!( + "Idunn route proof for {transaction_id} failed: {}", + route_proof_reason(error) + ), + ); } /// Write `detail` as the record's `last_error`, touching nothing else and @@ -8871,11 +8884,13 @@ impl Engine { ) { Ok(observation) => observation, Err(proof_error) if !rollback_allowed => { + self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return Err(proof_error).context( "incumbent authority was fenced; candidate route remains installed for fail-closed retry", ); } Err(proof_error) => { + self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return match driver.rollback( expected, &activation.runtime_instance_id, @@ -10560,8 +10575,15 @@ impl Engine { .iter() .any(|request| request.value.command_id == transaction.command_id); let cause = if now >= deadline.deadline_at_unix_millis { + // The step's latest failure is the reason the phase never + // finished, and the abort that ends it must not drop it. + let last_attempt = transaction + .last_error + .as_deref() + .map(|error| format!("; its last attempt failed: {error}")) + .unwrap_or_default(); format!( - "the {:?} phase passed its deadline at unix ms {}", + "the {:?} phase passed its deadline at unix ms {}{last_attempt}", transaction.phase, deadline.deadline_at_unix_millis ) } else if requested { @@ -11426,6 +11448,22 @@ fn runner_failure_journal_line( } } +fn route_proof_report_key(transaction_id: &str) -> String { + format!("route-proof:{transaction_id}") +} + +/// The stable route proof's failure in words: whether the listener said +/// nothing or answered wrongly, then the specific refusal (HTTP status line, +/// presence not active, challenge or authority disagreement). The texts it +/// carries name states and codes, never keys or signature bytes. +fn route_proof_reason(error: &anyhow::Error) -> String { + match error.downcast_ref::() { + Some(ChallengeFailure::Silent(_)) => format!("silent, nothing answered: {error:#}"), + Some(ChallengeFailure::Refused(_)) => format!("refused: {error:#}"), + None => format!("{error:#}"), + } +} + fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { @@ -16954,6 +16992,60 @@ mod tests { Ok(()) } + #[test] + fn a_route_proof_failure_names_what_the_listener_did() { + let reason = |error: anyhow::Error| route_proof_reason(&error); + let silent = anyhow::Error::new(ChallengeFailure::Silent(anyhow!( + "connecting stable CultNet HTTP route 127.0.0.1:9: refused" + ))); + let text = reason(silent); + assert!(text.contains("silent") && text.contains("127.0.0.1:9"), "{text}"); + + let status = anyhow::Error::new(ChallengeFailure::Refused(anyhow!( + "stable CultNet HTTP route rejected the snapshot challenge: HTTP/1.1 502 Bad Gateway" + ))) + .context("candidate did not answer its stable route challenge"); + let text = reason(status); + assert!(text.contains("refused") && text.contains("502 Bad Gateway"), "{text}"); + + let inactive = anyhow!("route proof runtime is warming, not active"); + assert!(reason(inactive).contains("warming, not active")); + + let disagrees = anyhow::Error::new(PresenceDisagrees { + disagreements: vec![OdinTopologyDisagreement { + code: "runtime-instance-mismatch".into(), + expected: Some("a".into()), + observed: Some("b".into()), + }], + }); + let text = reason(disagrees); + assert!(text.contains("runtime-instance-mismatch (expected a, observed b)"), "{text}"); + + let unbound = anyhow!("route proof response is not bound to the exact challenge"); + assert!(reason(unbound).contains("not bound to the exact challenge")); + } + + #[test] + fn a_route_proof_failure_is_reported_once_per_reason_and_cleared_on_recovery() -> Result<()> { + let world = EngineFixture::new()?; + let key = route_proof_report_key("tx-1"); + let said = |world: &EngineFixture| { + world + .engine + .fault_reports + .lock() + .unwrap() + .get(&key) + .and_then(|report| report.last.lock().unwrap().clone()) + }; + world.engine.report_route_proof_failure("tx-1", &anyhow!("route proof runtime is warming, not active")); + let first = said(&world).context("no report")?; + assert!(first.contains("tx-1") && first.contains("warming, not active"), "{first}"); + world.engine.clear_fault("tx-1"); + assert!(said(&world).is_none(), "recovery left the proof report armed"); + Ok(()) + } + #[test] fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { write_fixture_anchor()?; @@ -24257,9 +24349,39 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.run_to_routing()?; let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; routed.stub.hang_up.store(true, Ordering::SeqCst); - for _ in 0..3 { + // The Routing tick itself reports why the proof failed, once per + // distinct reason, and recovery clears it. + let proof_key = route_proof_report_key(&routed.transaction()?.transaction_id); + let said = |routed: &RoutedWorld| -> Option { + routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .get(&proof_key) + .and_then(|report| report.last.lock().unwrap().clone()) + }; + assert!(said(&routed).is_none(), "a proof report existed before any failure"); + routed.resume()?; + let first = said(&routed).context("the failing Routing tick reported no proof failure")?; + assert!(first.contains("route proof") && first.contains("silent, nothing answered"), "{first}"); + // The hung-up stub fails the same way each attempt, but the OS + // words the tail either as an early EOF or a broken pipe, so a + // retry may honestly report a second distinct reason. What may + // not change is everything up to that tail, and a repeat must not + // mint a new line each attempt. + let head = |text: &str| text.split("answered:").next().unwrap().to_owned(); + let mut seen = vec![first.clone()]; + for _ in 0..2 { routed.resume()?; + let again = said(&routed).context("a retry cleared the proof report")?; + assert_eq!(head(&again), head(&first), "a retry reworded the report"); + if !seen.contains(&again) { + seen.push(again); + } } + assert!(seen.len() <= 2, "the same failure was reported afresh on every retry: {seen:#?}"); // Before the deadline each attempt installs and rolls back, and no // resume moves the deadline. assert_eq!(routed.reloads(), 6); @@ -24302,6 +24424,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(records.len(), 1); assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + assert!(said(&routed).is_none(), "recovery left the proof report armed"); Ok(()) } @@ -24910,6 +25033,16 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .post_fencing_abort .context("the passed deadline did not end a deployment")?; assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!( + abort.error.contains("its last attempt failed"), + "the deadline abort dropped the proof's failure: {}", + abort.error + ); + assert!( + abort.error.contains("stable CultNet route stopped before its headers ended"), + "the deadline abort dropped the proof's own failure text: {}", + abort.error + ); assert!( routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), "the boot lift took a deployment's abort" From 8dd9f8d5036e256eb07676ec3226fcc23b63650e Mon Sep 17 00:00:00 2001 From: Eureka Date: Sat, 14 Nov 2026 02:23:49 +0000 Subject: [PATCH 04/10] Liveness gates presence: route challenge and topology refresh run only for an observed-running incarnation Cut the pre-liveness route and topology supervision and the pre-challenge lease verdict; re-insert both calls after the running-branch incident close, with report_once and clear_fault per target. Rewrite the dead-unit test, add four tests, point the routed fixture's lifecycle brake store into the world. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 256 ++++++++++++++++++++++++++++++++++++------- src/provisioning.rs | 2 +- 2 files changed, 215 insertions(+), 43 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 11db393..59560c1 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -7266,28 +7266,6 @@ impl Engine { || snapshot.on_target(¤t.value.target).any(|stored| { stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity }); - match self.supervise_admitted_route(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn rejected admitted {} route continuity: {error:#}", - current.value.target - ), - } - match self.refresh_admitted_topology(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved admitted {} after rejecting topology observation: {error:#}", - current.value.target - ), - } match self.retire_stale_incarnations(&snapshot, current) { Ok(true) => { progressed = true; @@ -7435,6 +7413,40 @@ impl Engine { ) }), ); + // Route and topology supervision are about a running incarnation; + // workload liveness decides whether there is one (route-continuity-cut F8). + let route_key = format!("route-supervision:{}", current.value.target); + match self.supervise_admitted_route(&snapshot, current) { + Ok(true) => { + self.clear_fault(&route_key); + progressed = true; + continue; + } + Ok(false) => self.clear_fault(&route_key), + Err(error) => self.report_once( + &route_key, + format!( + "Idunn rejected admitted {} route continuity: {error:#}", + current.value.target + ), + ), + } + let topology_key = format!("topology:{}", current.value.target); + match self.refresh_admitted_topology(&snapshot, current) { + Ok(true) => { + self.clear_fault(&topology_key); + progressed = true; + continue; + } + Ok(false) => self.clear_fault(&topology_key), + Err(error) => self.report_once( + &topology_key, + format!( + "Idunn preserved admitted {} after rejecting topology observation: {error:#}", + current.value.target + ), + ), + } continue; } let workload_error = operational_error @@ -7702,17 +7714,13 @@ impl Engine { "admitted route receipt names another incarnation" ); let binding = current.value.plan.parsed_inputs()?.1; - let lease_driver = if let Some(lease) = current.value.leasing.lease() { + let lease_driver = if current.value.leasing.lease().is_some() { let lease_path = &binding .process_write_lease .as_ref() .context("stateful admitted generation has no write-lease binding")? .record_path; let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - ensure!( - driver.observe_exact(lease)?, - "admitted process write lease is no longer exact" - ); Some(driver) } else { None @@ -15070,6 +15078,103 @@ mod tests { odin_reports_ready(world, transaction, sequence) } + /// An Odin-correlated generation, admitted over `workload`, with the + /// sequence of the reading it holds. + fn odin_admitted_world( + workload: Arc, + ) -> Result<(EngineFixture, DeploymentTransaction, u64)> { + let world = EngineFixture::with_workload(workload)?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let first = seeded_sequence(None) + 1; + odin_reports_ready(&world, &seeded, first)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + assert!(world.engine.retire_one_terminal_transaction()?); + Ok((world, seeded, first)) + } + + /// Odin's next reading, about an activation that is not the current one. + fn odin_republishes_another_activation( + world: &EngineFixture, + seeded: &DeploymentTransaction, + sequence: u64, + ) -> Result<()> { + let mut other = seeded.clone(); + other.activation.as_mut().context("no activation")?.runtime_instance_id = + sha256_id(b"another-incarnation"); + odin_republishes(world, &other, sequence) + } + + #[test] + fn a_dead_generation_does_not_take_odin_correlations() -> Result<()> { + let switch = SwitchWorkload::new(); + let (world, seeded, first) = odin_admitted_world(switch.clone())?; + switch.kill(); + let envelope = incumbent_envelope(&world)?; + + // A newer reading that binds the admitted activation, and one that does + // not: neither is taken, nor said, about an incarnation that is not there. + for reading in 0..2 { + if reading == 0 { + odin_republishes(&world, &seeded, first + 4)?; + } else { + odin_republishes_another_activation(&world, &seeded, first + 5)?; + } + for _ in 0..100 { + world.engine.supervise_one_admitted_generation()?; + } + assert_eq!(incumbent_envelope(&world)?, envelope); + assert!( + !world.engine.fault_reports.lock().unwrap().contains_key("topology:service"), + "a dead generation reported a topology rejection" + ); + } + Ok(()) + } + + #[test] + fn a_running_generation_with_a_non_binding_correlation_is_reported_once() -> Result<()> { + let (world, seeded, first) = odin_admitted_world(Arc::new(StillWorkload))?; + odin_republishes_another_activation(&world, &seeded, first + 4)?; + let envelope = incumbent_envelope(&world)?; + for _ in 0..50 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + assert_eq!(incumbent_envelope(&world)?, envelope); + let said = { + let reports = world.engine.fault_reports.lock().unwrap(); + reports + .get("topology:service") + .and_then(|report| report.last.lock().unwrap().clone()) + .context("the rejection was not said")? + }; + assert!(said.contains("after rejecting topology observation"), "{said}"); + // Said once while it lasts: the same fault offered again is silent. + world.engine.report_once("topology:service", said.clone()); + assert_eq!( + world.engine.fault_reports.lock().unwrap()["topology:service"].offer(Some(said)), + None + ); + + // A reading withdrawn ends the fault too: the key clears, so a + // recurrence would be said again. + let store = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); + assert!(store.delete_batch_if_unchanged(&store.pull_all_read_only_snapshot()?)?); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); + + // A binding reading is adopted, and leaves nothing said. + odin_reports_ready(&world, &seeded, first + 6)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); + let adopted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no generation")? + .value + .clone(); + assert_eq!(adopted.odin_publisher_sequence_cursor, first + 6); + Ok(()) + } + fn ready_sequence(transaction: &DeploymentTransaction) -> Option { transaction .ready @@ -22047,6 +22152,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .replace( "/etc/nginx/idunn-stream-routes/service.conf", &config.display().to_string().replace('\\', "/"), + ) + .replace( + "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", + &world.root.join("service-lifecycle-brake.cc").display().to_string(), ); let (binding, recipe_source) = if stateful { std::fs::create_dir_all(world.root.join("lease"))?; @@ -24161,30 +24270,93 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[cfg(unix)] #[test] - fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { + fn a_dead_unit_is_restarted_without_a_route_challenge() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.admit()?; routed.workload.kill(); routed.stub.hang_up.store(true, Ordering::SeqCst); routed.drift()?; + let before = admitted(&routed)?.route_supervision; + let reloads = routed.reloads(); + let survivals = meters_of(&routed.world, "service")?.route_survivals; - // Tick 1: the repair charges the target's meters and the proof - // fails. Both are writes, so they end the pass: nothing else is - // decided from the snapshot they made stale. - assert!(routed.tick()?); - let state = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(state.consecutive_failures, 1); - assert!(state.degraded_since_unix_millis.is_some()); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - let meters = meters_of(&routed.world, "service")?; - assert_eq!(meters.route_survivals.len(), 2); - assert!(meters.route_actuations.is_empty()); - - // Tick 2: the route waits, and the dead unit is restarted from a - // fresh read, counted in the same write that mints it. + // Tick 1: restart, not route repair, is the response to a dead + // incarnation. The continuity is minted and no route survival is + // charged for a proof nobody could have passed. assert!(routed.tick()?); assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); + assert_eq!(meters_of(&routed.world, "service")?.route_survivals, survivals); + assert_eq!(admitted(&routed)?.route_supervision, before); + assert_eq!(routed.reloads(), reloads, "the dead unit's route was repaired"); + Ok(()) + } + + /// Engage the target's lifecycle brake: the dead path may not mint. + fn engage_lifecycle_brake(routed: &RoutedWorld) -> Result<()> { + let generation = admitted(routed)?; + crate::provisioning::replace_lifecycle_brake( + &routed.world.root.join("service-lifecycle-brake.cc"), + IdunnLifecycleBrakeRecord { + schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), + authority: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), + runtime_id: generation.expected.runtime_id.clone(), + target: "service".into(), + scope: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), + status: "engaged".into(), + reason: "test".into(), + updated_at_unix_millis: now_millis()?, + released_until_unix_millis: None, + }, + ) + } + + #[cfg(unix)] + #[test] + fn a_parked_routed_target_is_not_route_supervised() -> Result<()> { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let granted_lease = routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(granted_lease); + routed.step()?; + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + routed.promote()?; + routed.drive_until(|transaction| transaction.is_terminal())?; + assert!(routed.world.engine.retire_one_terminal_transaction()?); + let generation = admitted(&routed)?; + let lease = generation.leasing.lease().context("no lease")?.clone(); + CultCacheWriteLeaseDriver::new( + "service", + routed.world.root.join("lease/process-write-lease.cc"), + ) + .revoke_exact(Some(&lease))?; + routed.workload.kill(); + engage_lifecycle_brake(&routed)?; + routed.drift()?; + let envelope = incumbent_envelope(&routed.world)?; + let reloads = routed.reloads(); + let programs = routed.world.route_stubs.count(""); + + for _ in 0..100 { + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "a parked target's route was reloaded"); + assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); + assert_eq!(incumbent_envelope(&routed.world)?, envelope); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + let reports = routed.world.engine.fault_reports.lock().unwrap(); + assert!( + !reports.keys().any(|key| key.starts_with("route-supervision:") || key.starts_with("route:")), + "{:?}", + reports.keys().collect::>() + ); Ok(()) } diff --git a/src/provisioning.rs b/src/provisioning.rs index 5af3903..324785e 100644 --- a/src/provisioning.rs +++ b/src/provisioning.rs @@ -304,7 +304,7 @@ fn lifecycle_brake_status(options: &BTreeMap) -> Result<()> { Ok(()) } -fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { +pub(crate) fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { record.validate()?; let store = SingleFileMessagePackBackingStore::new(path); let entries = store.pull_all_read_only_snapshot()?; From cdf987730e60df3a962a0e089af228fad61b998d Mon Sep 17 00:00:00 2001 From: Eureka Date: Sat, 14 Nov 2026 02:39:27 +0000 Subject: [PATCH 05/10] Routing proof waits out nginx's reload handoff and names who answered systemctl reload only signals nginx's master; old workers keep answering the stable listener until the new config is parsed and they are told to quit, so Idunn's first challenge could reach the incumbent and fail the attempt, rolling the fragment back. An answer naming the preflight's incumbent instance is now not-yet for 10 s (one challenge per 250 ms, never admitting, never actuating). Every failed proof now carries a RouteProofAnswer context: the runtime instance and activation signer ids the answer claimed (unauthenticated, ids only, 96 bytes each), so the journal line and abort last_error say who answered. Tests drive Routing through the stub runtime with incumbent-first, never-lets-go, foreign-activation and stranger replies. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 285 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 275 insertions(+), 10 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 11db393..f5c5490 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -8877,10 +8877,11 @@ impl Engine { rollback_allowed, &gate, )?; - let observation = match self.prove_stable_route( + let observation = match self.prove_routed_candidate_through_handoff( &admitted.value, &driver, membership_sha256, + preflight, ) { Ok(observation) => observation, Err(proof_error) if !rollback_allowed => { @@ -9805,6 +9806,42 @@ impl Engine { ) } + /// Routing's proof, waiting out nginx's reload handoff. `systemctl reload` + /// only signals the master: the old workers keep answering the stable + /// listener until the new configuration is parsed and they are told to + /// quit, so Idunn's first challenge can reach the incumbent. An answer + /// that names the preflight's incumbent is therefore not-yet for + /// `ROUTE_RELOAD_HANDOFF`; it can delay and never admit (the id is read + /// unauthenticated, and the candidate's proof is checked as before). + /// Anything else fails the attempt at once. One challenge per poll and + /// nothing else: no install, rollback, route gate or persistence. + fn prove_routed_candidate_through_handoff( + &self, + transaction: &DeploymentTransaction, + driver: &NginxRouteDriver, + membership_sha256: String, + preflight: &crate::drivers::RoutePreflightReceipt, + ) -> Result { + let candidate = &required(&transaction.activation, "route activation")?.runtime_instance_id; + let until = std::time::Instant::now() + ROUTE_RELOAD_HANDOFF; + loop { + match self.prove_stable_route(transaction, driver, membership_sha256.clone()) { + Err(error) + if preflight.changes_route() + && std::time::Instant::now() < until + && error.downcast_ref::().is_some_and(|answer| { + preflight.incumbent_runtime_instance_id.as_deref() + == Some(answer.answered_instance.as_str()) + && answer.answered_instance != *candidate + }) => + { + thread::sleep(ROUTE_RELOAD_HANDOFF_POLL); + } + result => return result, + } + } + } + fn prove_stable_route_against( &self, expected: &IdunnExpectedIncarnationRecord, @@ -9829,7 +9866,15 @@ impl Engine { challenged_at_unix_millis, received_at_unix_millis, &response.canonical_presence, - )?; + ) + .map_err(|error| match presence_answer_ids(&response.canonical_presence) { + Some((answered_instance, answered_activation_signer)) => error.context(RouteProofAnswer { + answered_instance: route_proof_label(&answered_instance), + answered_activation_signer: route_proof_label(&answered_activation_signer), + candidate_instance: route_proof_label(&activation.runtime_instance_id), + }), + None => error, + })?; let observation = RouteObservation { route_id: driver.binding.route_id.clone(), runtime_instance_id: activation.runtime_instance_id.clone(), @@ -11452,10 +11497,56 @@ fn route_proof_report_key(transaction_id: &str) -> String { format!("route-proof:{transaction_id}") } +/// How long Routing waits for nginx's old workers to let go of the stable +/// listener after a reload, derived from two ceilings: one Forward actuation +/// must suffice per attempt (12 an hour), and the 120 s Routing deadline must +/// keep room for at least four more backoff-spaced attempts if the route +/// genuinely never switches. +const ROUTE_RELOAD_HANDOFF: Duration = + if cfg!(test) { Duration::from_millis(400) } else { Duration::from_secs(10) }; +const ROUTE_RELOAD_HANDOFF_POLL: Duration = + if cfg!(test) { Duration::from_millis(20) } else { Duration::from_millis(250) }; + +/// Who answered a stable route challenge that failed authentication, as the +/// answer claimed itself: ids only, unauthenticated, never signature bytes. +/// A label on a rejection, and the one thing that may say "challenge again". +#[derive(Debug)] +struct RouteProofAnswer { + answered_instance: String, + answered_activation_signer: String, + candidate_instance: String, +} + +impl std::fmt::Display for RouteProofAnswer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "stable route answered as runtime instance {}, activation signer {} (unauthenticated); the candidate is {}", + self.answered_instance, self.answered_activation_signer, self.candidate_instance + ) + } +} + +/// The ids a presence claims, if it decodes at all. No re-encode check and no +/// validation: this labels a rejection and is never evidence. +fn presence_answer_ids(canonical: &[u8]) -> Option<(String, String)> { + let presence = + rmp_serde::from_slice::(canonical).ok()?; + Some((presence.runtime_instance_id, presence.activation_signer_identity_id)) +} + +/// An id cut to 96 bytes at a char boundary, control characters stripped. +fn route_proof_label(id: &str) -> String { + let id: String = id.chars().filter(|c| !c.is_control()).collect(); + id[..id.floor_char_boundary(96)].to_owned() +} + /// The stable route proof's failure in words: whether the listener said /// nothing or answered wrongly, then the specific refusal (HTTP status line, /// presence not active, challenge or authority disagreement). The texts it -/// carries name states and codes, never keys or signature bytes. +/// carries name states and codes, never keys or signature bytes. A rejected +/// answer names the instance and activation signer it claimed, unauthenticated, +/// ids only. fn route_proof_reason(error: &anyhow::Error) -> String { match error.downcast_ref::() { Some(ChallengeFailure::Silent(_)) => format!("silent, nothing answered: {error:#}"), @@ -21757,6 +21848,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation: IdunnRuntimeActivationRecord, provider: cultnet_rs::ServiceIdentitySigner, activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, + /// A second key the transaction's activation record never named. + other_signer: cultnet_rs::IdunnRuntimeActivationSigner, + /// The instance the stable listener answered as before the reload. + incumbent_instance: Mutex>, + stable_answers: AtomicUsize, state: Mutex<&'static str>, capacity: AtomicU32, sequence: AtomicU64, @@ -21786,11 +21882,36 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// A first Odin candidate answering its Warming challenge: the /// detail names the challenge as `idunn-warming:`. FirstOdinWarming, + /// The stable listener's first n answers come from the incumbent + /// (its instance id, a key the record never named), then Honest. + IncumbentFirst(usize), + /// The candidate's own instance id, signed with the wrong key. + ForeignActivation, + /// Neither the candidate nor the incumbent, wrong key. + StrangerInstance, } impl RuntimeStub { fn answer(&self, message_id: &str) -> Result> { + self.answer_on(message_id, false) + } + + fn answer_on(&self, message_id: &str, stable: bool) -> Result> { let expected = &self.expected; + let own = self.activation.runtime_instance_id.clone(); + let (instance, signer) = match *self.reply.lock().unwrap() { + Reply::IncumbentFirst(n) + if stable && self.stable_answers.fetch_add(1, Ordering::SeqCst) < n => + { + let incumbent = self.incumbent_instance.lock().unwrap().clone(); + (incumbent.context("no incumbent instance set")?, &self.other_signer) + } + Reply::ForeignActivation if stable => (own, &self.other_signer), + Reply::StrangerInstance if stable => { + (sha256_id(b"stranger instance"), &self.other_signer) + } + _ => (own, &self.activation_signer), + }; let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), target: expected.target.clone(), @@ -21802,7 +21923,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide state_schema_generation: expected.state_schema_generation.clone(), state_contract_sha256: expected.state_contract_sha256.clone(), runtime_id: expected.runtime_id.clone(), - runtime_instance_id: self.activation.runtime_instance_id.clone(), + runtime_instance_id: instance, bound_endpoint: expected .route .as_ref() @@ -21829,7 +21950,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide observed_at_unix_millis: now_millis()?, signature_algorithm: "ed25519".into(), signature: Vec::new(), - activation_signer_identity_id: self.activation_signer.identity_id(), + activation_signer_identity_id: signer.identity_id(), activation_signature: Vec::new(), }; let proof = presence.canonical_proof_payload()?; @@ -21837,8 +21958,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .provider .sign::(&proof) .signature; - presence.activation_signature = - self.activation_signer.sign_presence_proof(&presence)?; + presence.activation_signature = signer.sign_presence_proof(&presence)?; Ok(rmp_serde::to_vec(&presence)?) } @@ -21846,7 +21966,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide *self.state.lock().unwrap() = state; } - fn serve_one(&self, mut stream: TcpStream) -> Result<()> { + fn serve_one(&self, mut stream: TcpStream, stable: bool) -> Result<()> { stream.set_read_timeout(Some(Duration::from_secs(3)))?; let mut request = Vec::new(); let mut chunk = [0_u8; 4096]; @@ -21893,7 +22013,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide record_key: self.expected.target.clone(), stored_at: rfc3339_millis(now_millis()?)?, payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, - payload: self.answer(&message_id)?, + payload: self.answer_on(&message_id, stable)?, source_runtime_id: None, source_agent_id: None, source_role: None, @@ -21935,7 +22055,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide hook(); } if !stub.hang_up.load(Ordering::SeqCst) { - let _ = stub.serve_one(stream); + let _ = stub.serve_one(stream, !candidate); } } Err(_) => std::thread::sleep(Duration::from_millis(5)), @@ -22151,6 +22271,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide )?; let mut seed = Vec::new(); let activation = launch.write_credential(&mut seed)?; + let mut other_seed = Vec::new(); + IdunnRuntimeActivationLaunch::issue( + &expected, + sha256_id(b"unrecorded activation"), + now, + &world.engine.idunn_signer, + )? + .write_credential(&mut other_seed)?; let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; let mut workload = recorded.workload.clone().context("no recorded workload")?; match &mut workload { @@ -22182,6 +22310,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( &seed[..], )?, + other_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( + &other_seed[..], + )?, + incumbent_instance: Mutex::new(None), + stable_answers: AtomicUsize::new(0), state: Mutex::new("warming"), capacity: AtomicU32::new(minimum_capacity), sequence: AtomicU64::new(0), @@ -24428,6 +24561,138 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + // ----------------------------------------------------------------- + // The reload handoff: nginx's old workers may answer the stable + // listener for a moment after `systemctl reload` returns. + // ----------------------------------------------------------------- + + /// A deploy world at Routing whose preflight found an incumbent route + /// (another instance, another membership), so the install changes it. + #[cfg(unix)] + fn handoff_world(reply: Reply) -> Result<(RoutedWorld, String)> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + let incumbent = sha256_id(b"incumbent instance"); + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + let preflight = next.route_preflight.as_mut().context("no preflight")?; + preflight.incumbent_runtime_instance_id = Some(incumbent.clone()); + preflight.incumbent_membership_sha256 = Some(sha256_id(b"incumbent route\n")); + preflight.incumbent_configuration = Some(b"incumbent route\n".to_vec()); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + std::fs::write(routed.fragment_path(), b"incumbent route\n")?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + *routed.stub.incumbent_instance.lock().unwrap() = Some(incumbent.clone()); + *routed.stub.reply.lock().unwrap() = reply; + Ok((routed, incumbent)) + } + + fn forwards(routed: &RoutedWorld) -> Result { + Ok(meters_of(&routed.world, "service")?.route_actuations.len()) + } + + fn proof_report(routed: &RoutedWorld) -> Result> { + let key = route_proof_report_key(&routed.transaction()?.transaction_id); + Ok(routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .get(&key) + .and_then(|report| report.last.lock().unwrap().clone())) + } + + #[cfg(unix)] + #[test] + fn routing_waits_out_the_incumbent_answering_after_reload() -> Result<()> { + let (routed, _) = handoff_world(Reply::IncumbentFirst(3))?; + let before = routed.reloads(); + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + routed.promote()?; + assert_eq!(routed.reloads() - before, 1, "an install and no rollback"); + assert_eq!(forwards(&routed)?, 1); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 4); + assert!(proof_report(&routed)?.is_none(), "a route proof report remains"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn routing_fails_once_when_the_incumbent_never_lets_go() -> Result<()> { + let (routed, incumbent) = handoff_world(Reply::IncumbentFirst(usize::MAX))?; + let before = routed.reloads(); + let started = std::time::Instant::now(); + let error = routed.step().unwrap_err(); + assert!(started.elapsed() >= ROUTE_RELOAD_HANDOFF, "gave up before the handoff window"); + assert_eq!(routed.reloads() - before, 2, "install and one rollback"); + assert_eq!(forwards(&routed)?, 1); + let said = proof_report(&routed)?.context("no route proof report")?; + let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; + for text in [format!("{error:#}"), said] { + assert!(text.contains(&format!("runtime instance {incumbent}")), "{text}"); + assert!(text.contains(&format!("the candidate is {candidate}")), "{text}"); + } + routed.step().unwrap_err(); + assert_eq!(forwards(&routed)?, 2, "the second attempt charges one more Forward"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn routing_fails_at_once_when_the_candidate_answers_with_a_foreign_activation() -> Result<()> { + let (routed, _) = handoff_world(Reply::ForeignActivation)?; + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + let started = std::time::Instant::now(); + let error = routed.step().unwrap_err(); + assert!(started.elapsed() < ROUTE_RELOAD_HANDOFF); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + let text = format!("{error:#}"); + let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; + assert!(text.contains("runtime activation proof verification failed"), "{text}"); + assert!(text.contains(&format!("runtime instance {candidate}")), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn routing_fails_at_once_for_a_stranger_instance() -> Result<()> { + let (routed, _) = handoff_world(Reply::StrangerInstance)?; + let before = routed.reloads(); + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + let started = std::time::Instant::now(); + routed.step().unwrap_err(); + assert!(started.elapsed() < ROUTE_RELOAD_HANDOFF); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert_eq!(routed.reloads() - before, 2); + Ok(()) + } + + #[test] + fn route_proof_answer_names_ids_only() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let mut presence = rmp_serde::from_slice::( + &routed.stub.answer("probe")?, + )?; + presence.runtime_instance_id = format!("a\u{7}{}", "é".repeat(5_000)); + presence.activation_signer_identity_id = "b".repeat(10_240); + presence.signature = vec![0xAB; 64]; + presence.activation_signature = vec![0xCD; 64]; + let (instance, signer) = presence_answer_ids(&rmp_serde::to_vec(&presence)?) + .context("a decodable presence gave no ids")?; + let text = RouteProofAnswer { + answered_instance: route_proof_label(&instance), + answered_activation_signer: route_proof_label(&signer), + candidate_instance: route_proof_label(&"c".repeat(10_240)), + } + .to_string(); + assert!(text.len() <= 400, "{} bytes", text.len()); + assert!(!text.contains('\u{7}')); + assert!(!text.contains("abab") && !text.contains("cdcd")); + assert!(presence_answer_ids(b"not msgpack presence").is_none()); + Ok(()) + } + /// The live shape of F1. A continuity reuses the admitted Expected, so /// its membership is the admitted fragment already on disk: while its /// proof fails, the install and the rollback write nothing and reload From 47a9c3d89e5e8d1b4fa1b953dc75b88f3945cccf Mon Sep 17 00:00:00 2001 From: Eureka Date: Sat, 14 Nov 2026 03:02:47 +0000 Subject: [PATCH 06/10] Liveness cut: say lasting observe/lease/demotion faults once, mark a past-due route as waiting on a running incarnation Applies Soul findings to cut-liveness-gates-presence: report_once+clear_fault for the remaining per-tick eprintln sites in supervise_one_admitted_generation, status marks a past-due challenge, tests for the unreadable-fragment and HostUnobservable paths, F8 marked resolved in route-continuity-cut.md. Co-Authored-By: Claude Sonnet 5.5 --- docs/route-continuity-cut.md | 2 + src/control_plane.rs | 222 ++++++++++++++++++++++++++++------- 2 files changed, 180 insertions(+), 44 deletions(-) diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index d1f52d2..7b2573b 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -390,6 +390,8 @@ read-only command on Yggdrasil on 2026-09-29. Nothing was written to the host. (`:3670`, `:3715`), and topology refresh (`:3780`). After three refused continuity restarts, supervision stops restarting (`:3522-3528`) but keeps supervising the route of a dead incarnation. Read. + Resolved by `cut-liveness-gates-presence`: route challenge and topology + refresh now run only for an incarnation observed running. - F9. The continuity restart limit counts refused restarts per generation (`is_refused_restart`, `:3469-3488`). A restart that succeeds admits a new generation and resets the count. A unit that starts, is admitted, and then diff --git a/src/control_plane.rs b/src/control_plane.rs index 59560c1..40d5976 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5525,14 +5525,24 @@ fn render_supervision( } if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { lines.push(format!( - " route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}", + " route {} consecutive-failures {} last-challenge-at {}{} next-challenge-at {}{}", match route.degraded_since_unix_millis { Some(since) => format!("degraded-since {since}"), None => "healthy".to_owned(), }, route.consecutive_failures, at(route.last_challenge_at_unix_millis), - at(route.next_challenge_at_unix_millis) + route + .last_challenge_at_unix_millis + .map_or(String::new(), |last| format!(" ({} ms ago)", now.saturating_sub(last))), + at(route.next_challenge_at_unix_millis), + // Route supervision runs only for a running incarnation, so a + // challenge left past due is not a route that is being watched. + if route.next_challenge_at_unix_millis.is_some_and(|next| next < now) { + ", waits on a running incarnation" + } else { + "" + } )); } if generation.is_none_or(|value| value.route_supervision.is_some()) @@ -5684,6 +5694,11 @@ struct Engine { /// One `ReportOnce` per faulting record, so one wedged transaction says /// its fault once and never drowns another's. fault_reports: Mutex>, + /// The key of every `report_once` line that reached stderr: what a test + /// counts, since stderr is not observable from inside it, and a key's + /// own entry is dropped when its fault clears. + #[cfg(test)] + printed_reports: Mutex>, /// Retry pacing for a transaction whose step keeps failing, by /// transaction id: consecutive failures and the earliest next attempt. /// Process-local on purpose: a restart retries at once, and no schema @@ -5811,6 +5826,8 @@ impl Engine { runner_stand_in: Mutex::default(), history_report: ReportOnce::default(), fault_reports: Mutex::default(), + #[cfg(test)] + printed_reports: Mutex::default(), resume_backoff: Mutex::default(), lift_held: Mutex::default(), lift_unsettled: std::sync::atomic::AtomicBool::new(false), @@ -5929,6 +5946,8 @@ impl Engine { .or_default() .offer(Some(message)); if let Some(message) = offered { + #[cfg(test)] + self.printed_reports.lock().unwrap().push(key.to_string()); eprintln!("{message}"); } } @@ -7271,10 +7290,13 @@ impl Engine { progressed = true; continue; } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved a stale {} incarnation projection: {error:#}", - current.value.target + Ok(false) => self.clear_fault(&format!("stale-incarnation:{}", current.value.target)), + Err(error) => self.report_once( + &format!("stale-incarnation:{}", current.value.target), + format!( + "Idunn preserved a stale {} incarnation projection: {error:#}", + current.value.target + ), ), } // Restore the admitted Expected before anything is asked of the @@ -7303,11 +7325,16 @@ impl Engine { } Ok(()) })(); - if let Err(error) = repair { - eprintln!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target - ); + let repair_key = format!("projection-repair:{}", current.value.target); + match repair { + Ok(()) => self.clear_fault(&repair_key), + Err(error) => self.report_once( + &repair_key, + format!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ), + ), } let mut operational_error = None; let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { @@ -7318,6 +7345,7 @@ impl Engine { ) }) { Ok(observation) => { + self.clear_fault(&format!("observe:{}", current.value.target)); let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { let lease_health = (|| -> Result { let lease_path = current @@ -7338,6 +7366,10 @@ impl Engine { } bail!("write-lease store contains unexpected authority") })(); + let lease_key = format!("write-lease:{}", current.value.target); + if lease_health.is_ok() { + self.clear_fault(&lease_key); + } match lease_health { Ok(true) => false, Ok(false) => { @@ -7346,9 +7378,12 @@ impl Engine { true } Err(error) => { - eprintln!( - "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", - current.value.target + self.report_once( + &lease_key, + format!( + "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", + current.value.target + ), ); continue; } @@ -7368,13 +7403,20 @@ impl Engine { // silence, not a death; counting it as one burned every // continuity attempt in the reattach window on // 2026-09-11 and left a healthy Muninn demoted. - eprintln!( - "Idunn cannot observe admitted {} this tick: {error:#}", - current.value.target + // Said once while it lasts, and cleared the moment the host + // answers: a reattach window is not a line per tick. + self.report_once( + &format!("observe:{}", current.value.target), + format!( + "Idunn cannot observe admitted {} this tick: {error:#}", + current.value.target + ), ); continue; } Err(error) => { + // The host answered, with a death: no longer silence. + self.clear_fault(&format!("observe:{}", current.value.target)); operational_error = Some(error); None } @@ -7414,7 +7456,9 @@ impl Engine { }), ); // Route and topology supervision are about a running incarnation; - // workload liveness decides whether there is one (route-continuity-cut F8). + // workload liveness decides whether there is one (route-continuity-cut F8, + // resolved by this order). Topology cannot bind without an observed + // activation, so a HostUnobservable tick skips both too. let route_key = format!("route-supervision:{}", current.value.target); match self.supervise_admitted_route(&snapshot, current) { Ok(true) => { @@ -7584,10 +7628,17 @@ impl Engine { // could not resolve. So a failed demotion mints nothing. The // failure is written to the generation, and the next attempt is // not before the deferral ends. + let demotion_key = format!("demotion:{}", current.value.target); + if demotion.is_ok() { + self.clear_fault(&demotion_key); + } if let Err(error) = demotion { - eprintln!( - "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", - current.value.target + self.report_once( + &demotion_key, + format!( + "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", + current.value.target + ), ); let mut deferred = supervision.clone(); deferred.defer_continuity(now, &bounded_detail(&format!("{error:#}"))); @@ -15131,6 +15182,10 @@ mod tests { Ok(()) } + fn printed(world: &EngineFixture, key: &str) -> usize { + world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() + } + #[test] fn a_running_generation_with_a_non_binding_correlation_is_reported_once() -> Result<()> { let (world, seeded, first) = odin_admitted_world(Arc::new(StillWorkload))?; @@ -15140,30 +15195,12 @@ mod tests { assert!(!world.engine.supervise_one_admitted_generation()?); } assert_eq!(incumbent_envelope(&world)?, envelope); - let said = { - let reports = world.engine.fault_reports.lock().unwrap(); - reports - .get("topology:service") - .and_then(|report| report.last.lock().unwrap().clone()) - .context("the rejection was not said")? - }; - assert!(said.contains("after rejecting topology observation"), "{said}"); - // Said once while it lasts: the same fault offered again is silent. - world.engine.report_once("topology:service", said.clone()); - assert_eq!( - world.engine.fault_reports.lock().unwrap()["topology:service"].offer(Some(said)), - None - ); + // The engine said it once over fifty ticks. + assert_eq!(printed(&world, "topology:service"), 1); + assert!(world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); - // A reading withdrawn ends the fault too: the key clears, so a - // recurrence would be said again. - let store = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); - assert!(store.delete_batch_if_unchanged(&store.pull_all_read_only_snapshot()?)?); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); - - // A binding reading is adopted, and leaves nothing said. - odin_reports_ready(&world, &seeded, first + 6)?; + // A binding reading is adopted and ends the fault, so the key clears. + odin_republishes(&world, &seeded, first + 6)?; assert!(world.engine.supervise_one_admitted_generation()?); assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); let adopted = ControlSnapshot::read(&world.state_store)? @@ -15172,6 +15209,17 @@ mod tests { .value .clone(); assert_eq!(adopted.odin_publisher_sequence_cursor, first + 6); + + // A recurrence is said again, once; a reading withdrawn ends it too. + odin_republishes_another_activation(&world, &seeded, first + 8)?; + for _ in 0..20 { + world.engine.supervise_one_admitted_generation()?; + } + assert_eq!(printed(&world, "topology:service"), 2); + let store = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); + assert!(store.delete_batch_if_unchanged(&store.pull_all_read_only_snapshot()?)?); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); Ok(()) } @@ -21763,6 +21811,24 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + #[test] + fn status_says_a_route_left_past_due_waits_on_a_running_incarnation() -> Result<()> { + let now = 10_000_000_u64; + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.route_supervision = Some(RouteSupervisionState { + last_challenge_at_unix_millis: Some(now - 5_000), + next_challenge_at_unix_millis: Some(now - 4_000), + ..RouteSupervisionState::default() + }); + let lines = render_supervision("service", Some(&generation), None, now).join("\n"); + assert!(lines.contains("last-challenge-at 9995000 (5000 ms ago)"), "{lines}"); + assert!(lines.contains("waits on a running incarnation"), "{lines}"); + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(now + 1); + let lines = render_supervision("service", Some(&generation), None, now).join("\n"); + assert!(!lines.contains("waits on a running incarnation"), "{lines}"); + Ok(()) + } + #[test] fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { let now = 10_000_000_u64; @@ -24360,6 +24426,74 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + fn printed_in(routed: &RoutedWorld, key: &str) -> usize { + routed.world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() + } + + #[cfg(unix)] + #[test] + fn a_running_route_whose_fragment_cannot_be_read_is_reported_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let key = "route-supervision:service"; + let fragment = routed.fragment_path(); + let bytes = std::fs::read(&fragment)?; + // The fragment is a directory: reading it fails, and nothing is + // written, so every tick meets the same error. + std::fs::remove_file(&fragment)?; + std::fs::create_dir(&fragment)?; + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, key), 1); + assert!(routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); + + // The fragment is back: the fault ends and the key clears. + std::fs::remove_dir(&fragment)?; + std::fs::write(&fragment, &bytes)?; + routed.tick()?; + assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); + + // A recurrence is said again, once. + std::fs::remove_file(&fragment)?; + std::fs::create_dir(&fragment)?; + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, key), 2); + Ok(()) + } + + /// Skipping route and topology on a tick the host could not be asked is + /// the steward's ruling (no activation can be bound without an + /// observation; no routed target runs on a host actuator today). + #[cfg(unix)] + #[test] + fn a_host_that_cannot_be_asked_skips_route_supervision_and_is_said_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.drift()?; + let envelope = incumbent_envelope(&routed.world)?; + let reloads = routed.reloads(); + routed.workload.silence(true); + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "route supervision ran without an observation"); + assert_eq!(incumbent_envelope(&routed.world)?, envelope); + assert_eq!(printed_in(&routed, "observe:service"), 1); + + routed.workload.silence(false); + routed.tick()?; + assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key("observe:service")); + routed.workload.silence(true); + for _ in 0..5 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, "observe:service"), 2); + Ok(()) + } + #[cfg(unix)] #[test] fn a_proved_challenge_clears_the_degradation() -> Result<()> { From c30f89f0dfcb47a02f2da001506eb44976b88a44 Mon Sep 17 00:00:00 2001 From: Eureka Date: Sat, 14 Nov 2026 03:19:57 +0000 Subject: [PATCH 07/10] Pin the route handoff window, cadence, signer label and unchanged-route guard The handoff loop reads its clock through a test-held virtual clock, so the window and poll cadence are asserted exactly and no host load moves them; the fails-at-once tests assert through stable_hits and the absence of sleeps. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 126 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 114 insertions(+), 12 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index f5c5490..33df428 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -9823,19 +9823,19 @@ impl Engine { preflight: &crate::drivers::RoutePreflightReceipt, ) -> Result { let candidate = &required(&transaction.activation, "route activation")?.runtime_instance_id; - let until = std::time::Instant::now() + ROUTE_RELOAD_HANDOFF; + let until = handoff_now() + ROUTE_RELOAD_HANDOFF; loop { match self.prove_stable_route(transaction, driver, membership_sha256.clone()) { Err(error) if preflight.changes_route() - && std::time::Instant::now() < until + && handoff_now() < until && error.downcast_ref::().is_some_and(|answer| { preflight.incumbent_runtime_instance_id.as_deref() == Some(answer.answered_instance.as_str()) && answer.answered_instance != *candidate }) => { - thread::sleep(ROUTE_RELOAD_HANDOFF_POLL); + handoff_sleep(ROUTE_RELOAD_HANDOFF_POLL); } result => return result, } @@ -11507,6 +11507,47 @@ const ROUTE_RELOAD_HANDOFF: Duration = const ROUTE_RELOAD_HANDOFF_POLL: Duration = if cfg!(test) { Duration::from_millis(20) } else { Duration::from_millis(250) }; +/// The handoff loop's clock. Production reads the real one; a test may install +/// a virtual one on its own thread, so the window and the poll cadence are +/// measured exactly, whatever the host load. +#[cfg(not(test))] +fn handoff_now() -> std::time::Instant { + std::time::Instant::now() +} +#[cfg(not(test))] +fn handoff_sleep(pause: Duration) { + thread::sleep(pause); +} +#[cfg(test)] +thread_local! { + /// (virtual offset, sleeps taken, clock reads) while a test holds it. + static HANDOFF_CLOCK: std::cell::RefCell, usize)>> = + const { std::cell::RefCell::new(None) }; +} +#[cfg(test)] +fn handoff_now() -> std::time::Instant { + static BASE: std::sync::OnceLock = std::sync::OnceLock::new(); + let base = *BASE.get_or_init(std::time::Instant::now); + HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { + Some((offset, _, reads)) => { + *reads += 1; + assert!(*reads < 10_000, "the handoff loop spins without sleeping"); + base + *offset + } + None => std::time::Instant::now(), + }) +} +#[cfg(test)] +fn handoff_sleep(pause: Duration) { + HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { + Some((offset, sleeps, _)) => { + *offset += pause; + sleeps.push(pause); + } + None => thread::sleep(pause), + }) +} + /// Who answered a stable route challenge that failed authentication, as the /// answer claimed itself: ids only, unauthenticated, never signature bytes. /// A label on a rejection, and the one thing that may say "challenge again". @@ -24570,16 +24611,30 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// (another instance, another membership), so the install changes it. #[cfg(unix)] fn handoff_world(reply: Reply) -> Result<(RoutedWorld, String)> { + handoff_world_with(reply, false) + } + + /// `same_membership`: the incumbent's membership equals the + /// candidate's, so the install changes no route. + #[cfg(unix)] + fn handoff_world_with(reply: Reply, same_membership: bool) -> Result<(RoutedWorld, String)> { let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; let incumbent = sha256_id(b"incumbent instance"); let current = resident(&routed.world)?; let mut next = current.value.clone(); + let mut on_disk = b"incumbent route\n".to_vec(); + if same_membership { + on_disk = NginxRouteDriver::new( + next.plan.as_ref().unwrap().parsed_inputs()?.1.route.context("no route binding")?, + ) + .render(next.expected.as_ref().context("no Expected")?)?; + } let preflight = next.route_preflight.as_mut().context("no preflight")?; preflight.incumbent_runtime_instance_id = Some(incumbent.clone()); - preflight.incumbent_membership_sha256 = Some(sha256_id(b"incumbent route\n")); - preflight.incumbent_configuration = Some(b"incumbent route\n".to_vec()); + preflight.incumbent_membership_sha256 = Some(sha256_id(&on_disk)); + preflight.incumbent_configuration = Some(on_disk.clone()); replace_transaction(&routed.world.state_store, ¤t, &next)?; - std::fs::write(routed.fragment_path(), b"incumbent route\n")?; + std::fs::write(routed.fragment_path(), &on_disk)?; routed.stub.set_state("active"); routed.run_to_routing()?; *routed.stub.incumbent_instance.lock().unwrap() = Some(incumbent.clone()); @@ -24603,9 +24658,32 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .and_then(|report| report.last.lock().unwrap().clone())) } + /// Holds a virtual handoff clock on this thread: the wait is measured + /// in the loop's own sleeps, so no host load can move it. + struct VirtualHandoffClock; + + impl VirtualHandoffClock { + fn hold() -> Self { + HANDOFF_CLOCK.with(|c| *c.borrow_mut() = Some((Duration::ZERO, Vec::new(), 0))); + Self + } + + /// The pauses the loop took so far. + fn sleeps(&self) -> Vec { + HANDOFF_CLOCK.with(|c| c.borrow().as_ref().unwrap().1.clone()) + } + } + + impl Drop for VirtualHandoffClock { + fn drop(&mut self) { + HANDOFF_CLOCK.with(|c| *c.borrow_mut() = None); + } + } + #[cfg(unix)] #[test] fn routing_waits_out_the_incumbent_answering_after_reload() -> Result<()> { + let clock = VirtualHandoffClock::hold(); let (routed, _) = handoff_world(Reply::IncumbentFirst(3))?; let before = routed.reloads(); let hits = routed.stub.stable_hits.load(Ordering::SeqCst); @@ -24613,6 +24691,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(routed.reloads() - before, 1, "an install and no rollback"); assert_eq!(forwards(&routed)?, 1); assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 4); + assert_eq!(clock.sleeps(), vec![ROUTE_RELOAD_HANDOFF_POLL; 3]); assert!(proof_report(&routed)?.is_none(), "a route proof report remains"); Ok(()) } @@ -24620,17 +24699,26 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[cfg(unix)] #[test] fn routing_fails_once_when_the_incumbent_never_lets_go() -> Result<()> { + let clock = VirtualHandoffClock::hold(); let (routed, incumbent) = handoff_world(Reply::IncumbentFirst(usize::MAX))?; let before = routed.reloads(); - let started = std::time::Instant::now(); + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); let error = routed.step().unwrap_err(); - assert!(started.elapsed() >= ROUTE_RELOAD_HANDOFF, "gave up before the handoff window"); + // The window is exactly ROUTE_RELOAD_HANDOFF of polls at exactly + // the poll cadence: one challenge before each pause and one after + // the last. + let sleeps = clock.sleeps(); + let polls = (ROUTE_RELOAD_HANDOFF.as_millis() / ROUTE_RELOAD_HANDOFF_POLL.as_millis()) as usize; + assert_eq!(sleeps, vec![ROUTE_RELOAD_HANDOFF_POLL; polls]); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, polls + 1); assert_eq!(routed.reloads() - before, 2, "install and one rollback"); assert_eq!(forwards(&routed)?, 1); let said = proof_report(&routed)?.context("no route proof report")?; let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; + let signer = routed.stub.other_signer.identity_id(); for text in [format!("{error:#}"), said] { assert!(text.contains(&format!("runtime instance {incumbent}")), "{text}"); + assert!(text.contains(&format!("activation signer {signer}")), "{text}"); assert!(text.contains(&format!("the candidate is {candidate}")), "{text}"); } routed.step().unwrap_err(); @@ -24641,12 +24729,12 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[cfg(unix)] #[test] fn routing_fails_at_once_when_the_candidate_answers_with_a_foreign_activation() -> Result<()> { + let clock = VirtualHandoffClock::hold(); let (routed, _) = handoff_world(Reply::ForeignActivation)?; let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - let started = std::time::Instant::now(); let error = routed.step().unwrap_err(); - assert!(started.elapsed() < ROUTE_RELOAD_HANDOFF); assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert!(clock.sleeps().is_empty()); let text = format!("{error:#}"); let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; assert!(text.contains("runtime activation proof verification failed"), "{text}"); @@ -24657,17 +24745,31 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[cfg(unix)] #[test] fn routing_fails_at_once_for_a_stranger_instance() -> Result<()> { + let clock = VirtualHandoffClock::hold(); let (routed, _) = handoff_world(Reply::StrangerInstance)?; let before = routed.reloads(); let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - let started = std::time::Instant::now(); routed.step().unwrap_err(); - assert!(started.elapsed() < ROUTE_RELOAD_HANDOFF); assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert!(clock.sleeps().is_empty()); assert_eq!(routed.reloads() - before, 2); Ok(()) } + /// An incumbent whose membership equals the candidate's has no route + /// to hand off, so its answer is a plain failure, never waited on. + #[cfg(unix)] + #[test] + fn routing_does_not_wait_when_the_route_is_unchanged() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, _) = handoff_world_with(Reply::IncumbentFirst(1), true)?; + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + routed.step().unwrap_err(); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert!(clock.sleeps().is_empty()); + Ok(()) + } + #[test] fn route_proof_answer_names_ids_only() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; From eace76870ad592174e83db1ba9237617e0a9efe6 Mon Sep 17 00:00:00 2001 From: Eureka Date: Sat, 14 Nov 2026 03:38:14 +0000 Subject: [PATCH 08/10] Pin liveness cut: truthful past-due status, dead-topology test reaches its gate, report-once and empty-lease pins Status says how overdue a route challenge is, not a cause it cannot see. The dead-generation topology test engages the lifecycle brake so every tick reaches the gate. New pins: observe/lease/demotion/stale-incarnation/projection repair said once and cleared; a running target with an empty lease is not route challenged. Co-Authored-By: Claude Sonnet 5.5 --- src/control_plane.rs | 268 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 234 insertions(+), 34 deletions(-) diff --git a/src/control_plane.rs b/src/control_plane.rs index 40d5976..eb9761b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5525,8 +5525,7 @@ fn render_supervision( } if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { lines.push(format!( - " route {} consecutive-failures {} last-challenge-at {}{} next-challenge-at {}{}", - match route.degraded_since_unix_millis { + " route {} consecutive-failures {} last-challenge-at {}{} next-challenge-at {}{}", match route.degraded_since_unix_millis { Some(since) => format!("degraded-since {since}"), None => "healthy".to_owned(), }, @@ -5536,13 +5535,13 @@ fn render_supervision( .last_challenge_at_unix_millis .map_or(String::new(), |last| format!(" ({} ms ago)", now.saturating_sub(last))), at(route.next_challenge_at_unix_millis), - // Route supervision runs only for a running incarnation, so a - // challenge left past due is not a route that is being watched. - if route.next_challenge_at_unix_millis.is_some_and(|next| next < now) { - ", waits on a running incarnation" - } else { - "" - } + // Status sees the stored schedule, not why it is not being kept + // (a refused write, a transaction, a daemon that is down): it + // says only how overdue the challenge is. + route + .next_challenge_at_unix_millis + .filter(|next| *next < now) + .map_or(String::new(), |next| format!(" (past due {} ms)", now - next)) )); } if generation.is_none_or(|value| value.route_supervision.is_some()) @@ -14226,6 +14225,10 @@ mod tests { .replace( "/var/lib/gamecult/idunn/service-deployment-brake.cc", &world.root.join("service-deployment-brake.cc").display().to_string(), + ) + .replace( + "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", + &world.root.join("service-lifecycle-brake.cc").display().to_string(), ), )?; let recipe = RECIPE @@ -15155,11 +15158,39 @@ mod tests { odin_republishes(world, &other, sequence) } + /// Engage the admitted target's lifecycle brake, at the path its seeded + /// binding names: a dead incarnation mints no continuity, so every later + /// tick reaches the same gates. + fn engage_lifecycle_brake_in(world: &EngineFixture) -> Result<()> { + let generation = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .value + .clone(); + crate::provisioning::replace_lifecycle_brake( + &world.root.join("service-lifecycle-brake.cc"), + IdunnLifecycleBrakeRecord { + schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), + authority: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), + runtime_id: generation.expected.runtime_id.clone(), + target: "service".into(), + scope: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), + status: "engaged".into(), + reason: "test".into(), + updated_at_unix_millis: now_millis()?, + released_until_unix_millis: None, + }, + ) + } + #[test] fn a_dead_generation_does_not_take_odin_correlations() -> Result<()> { let switch = SwitchWorkload::new(); let (world, seeded, first) = odin_admitted_world(switch.clone())?; switch.kill(); + // No continuity is minted, so no transaction ever owns the target and + // every tick reaches the topology gate. + engage_lifecycle_brake_in(&world)?; let envelope = incumbent_envelope(&world)?; // A newer reading that binds the admitted activation, and one that does @@ -15173,6 +15204,10 @@ mod tests { for _ in 0..100 { world.engine.supervise_one_admitted_generation()?; } + assert!( + ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), + "a transaction owns the target: later ticks never reach the gate" + ); assert_eq!(incumbent_envelope(&world)?, envelope); assert!( !world.engine.fault_reports.lock().unwrap().contains_key("topology:service"), @@ -21812,7 +21847,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide } #[test] - fn status_says_a_route_left_past_due_waits_on_a_running_incarnation() -> Result<()> { + fn status_says_how_overdue_a_route_challenge_is_and_claims_no_cause() -> Result<()> { let now = 10_000_000_u64; let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; generation.route_supervision = Some(RouteSupervisionState { @@ -21822,10 +21857,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide }); let lines = render_supervision("service", Some(&generation), None, now).join("\n"); assert!(lines.contains("last-challenge-at 9995000 (5000 ms ago)"), "{lines}"); - assert!(lines.contains("waits on a running incarnation"), "{lines}"); - generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(now + 1); + assert!(lines.contains("next-challenge-at 9996000 (past due 4000 ms)"), "{lines}"); + assert!(!lines.contains("waits on"), "{lines}"); + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(now); let lines = render_supervision("service", Some(&generation), None, now).join("\n"); - assert!(!lines.contains("waits on a running incarnation"), "{lines}"); + assert!(!lines.contains("past due"), "{lines}"); Ok(()) } @@ -24358,28 +24394,9 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } - /// Engage the target's lifecycle brake: the dead path may not mint. - fn engage_lifecycle_brake(routed: &RoutedWorld) -> Result<()> { - let generation = admitted(routed)?; - crate::provisioning::replace_lifecycle_brake( - &routed.world.root.join("service-lifecycle-brake.cc"), - IdunnLifecycleBrakeRecord { - schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), - authority: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), - runtime_id: generation.expected.runtime_id.clone(), - target: "service".into(), - scope: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), - status: "engaged".into(), - reason: "test".into(), - updated_at_unix_millis: now_millis()?, - released_until_unix_millis: None, - }, - ) - } - + /// A target with a process write lease, admitted and running. #[cfg(unix)] - #[test] - fn a_parked_routed_target_is_not_route_supervised() -> Result<()> { + fn admitted_stateful_world() -> Result { let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; let granted_lease = routed @@ -24396,6 +24413,17 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.promote()?; routed.drive_until(|transaction| transaction.is_terminal())?; assert!(routed.world.engine.retire_one_terminal_transaction()?); + Ok(routed) + } + + /// Engage the target's lifecycle brake: the dead path may not mint. + fn engage_lifecycle_brake(routed: &RoutedWorld) -> Result<()> { + engage_lifecycle_brake_in(&routed.world) + } + #[cfg(unix)] + #[test] + fn a_parked_routed_target_is_not_route_supervised() -> Result<()> { + let routed = admitted_stateful_world()?; let generation = admitted(&routed)?; let lease = generation.leasing.lease().context("no lease")?.clone(); CultCacheWriteLeaseDriver::new( @@ -24426,6 +24454,178 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + fn fault_keys(routed: &RoutedWorld) -> Vec { + routed.world.engine.fault_reports.lock().unwrap().keys().cloned().collect() + } + + /// Soul's probe: a route record that cannot be written leaves the + /// challenge past due on a running incarnation. Status shows the age + /// and does not name a cause it cannot see. + #[cfg(unix)] + #[test] + fn status_of_a_past_due_route_on_a_running_incarnation_claims_no_wait() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.drift()?; + assert!(routed.tick()?); + assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); + routed.stub.hang_up.store(false, Ordering::SeqCst); + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); + })?; + let fragment = routed.fragment_path(); + std::fs::remove_file(&fragment)?; + std::fs::create_dir(&fragment)?; + for _ in 0..20 { + routed.tick()?; + } + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + assert!(fault_keys(&routed).contains(&"route-supervision:service".to_owned())); + let generation = admitted(&routed)?; + let lines = render_supervision("service", Some(&generation), None, now_millis()?).join("\n"); + assert!(lines.contains("(past due "), "{lines}"); + assert!(!lines.contains("waits on"), "{lines}"); + Ok(()) + } + + /// A target whose lease store is empty while its process runs is a + /// dead-path case: the route is not challenged for it. + #[cfg(unix)] + #[test] + fn a_running_target_with_an_empty_lease_is_not_route_challenged() -> Result<()> { + let routed = admitted_stateful_world()?; + let lease = admitted(&routed)?.leasing.lease().context("no lease")?.clone(); + CultCacheWriteLeaseDriver::new( + "service", + routed.world.root.join("lease/process-write-lease.cc"), + ) + .revoke_exact(Some(&lease))?; + engage_lifecycle_brake(&routed)?; + routed.drift()?; + let envelope = incumbent_envelope(&routed.world)?; + let reloads = routed.reloads(); + let programs = routed.world.route_stubs.count(""); + for _ in 0..50 { + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "an empty-lease target's route was reloaded"); + assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); + assert_eq!(incumbent_envelope(&routed.world)?, envelope); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + assert!( + !fault_keys(&routed).iter().any(|key| key.starts_with("route")), + "{:?}", + fault_keys(&routed) + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_host_that_answers_with_a_death_ends_the_silence_fault() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + engage_lifecycle_brake(&routed)?; + routed.workload.silence(true); + for _ in 0..5 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, "observe:service"), 1); + routed.workload.silence(false); + routed.workload.kill(); + routed.tick()?; + assert!(!fault_keys(&routed).contains(&"observe:service".to_owned())); + Ok(()) + } + + /// Replace the topology store with a directory: every read of it fails. + #[cfg(unix)] + fn break_topology(routed: &RoutedWorld) -> Result<()> { + let store = routed.world.root.join("topology.cc"); + if store.exists() { + std::fs::rename(&store, routed.world.root.join("topology.aside"))?; + } + std::fs::create_dir(&store)?; + Ok(()) + } + + #[cfg(unix)] + fn mend_topology(routed: &RoutedWorld) -> Result<()> { + let store = routed.world.root.join("topology.cc"); + std::fs::remove_dir(&store)?; + let aside = routed.world.root.join("topology.aside"); + if aside.exists() { + std::fs::rename(aside, &store)?; + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_stale_incarnation_and_a_projection_repair_that_fail_are_each_said_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let keys = ["stale-incarnation:service", "projection-repair:service"]; + for round in 1..=2 { + break_topology(&routed)?; + for _ in 0..30 { + routed.tick()?; + } + for key in keys { + assert_eq!(printed_in(&routed, key), round, "{key}"); + assert!(fault_keys(&routed).contains(&key.to_owned()), "{key}"); + } + mend_topology(&routed)?; + routed.tick()?; + for key in keys { + assert!(!fault_keys(&routed).contains(&key.to_owned()), "{key}"); + } + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_refused_demotion_is_said_once_and_cleared_when_it_succeeds() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.workload.kill(); + break_topology(&routed)?; + // Each attempt is let through by forgetting the deferral it wrote. + for _ in 0..6 { + set_meters(&routed.world, &TargetSupervision::new("service"))?; + routed.tick()?; + } + assert_eq!(printed_in(&routed, "demotion:service"), 1); + assert!(fault_keys(&routed).contains(&"demotion:service".to_owned())); + mend_topology(&routed)?; + set_meters(&routed.world, &TargetSupervision::new("service"))?; + routed.tick()?; + assert!(!fault_keys(&routed).contains(&"demotion:service".to_owned())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unsafe_write_lease_store_is_said_once_and_cleared_when_it_is_exact() -> Result<()> { + let routed = admitted_stateful_world()?; + let store = routed.world.root.join("lease/process-write-lease.cc"); + let exact = std::fs::read(&store)?; + for round in 1..=2 { + std::fs::write(&store, b"not a lease store")?; + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, "write-lease:service"), round); + assert!(fault_keys(&routed).contains(&"write-lease:service".to_owned())); + std::fs::write(&store, &exact)?; + routed.tick()?; + assert!(!fault_keys(&routed).contains(&"write-lease:service".to_owned())); + } + Ok(()) + } + fn printed_in(routed: &RoutedWorld, key: &str) -> usize { routed.world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() } From 6c34758a30cdc34c829e9e32272331326bf880ad Mon Sep 17 00:00:00 2001 From: Eureka Date: Sat, 14 Nov 2026 08:21:12 +0000 Subject: [PATCH 09/10] Static release carry: explicit 0444 under UMask 0027 The carry file was created with .mode(0o444), which the service's UMask=0027 masks to 0440; www-data could not read it and the sealed-tree verifier rejected it. Set the permissions on the open handle before the rename, and prove it in a child process under umask 0027. Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/drivers.rs b/src/drivers.rs index 46cb916..714c460 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -762,7 +762,7 @@ impl StaticReleasePort for FilesystemStaticRelease { binding: &StaticReleaseBinding, release_id: &str, ) -> Result { - use std::os::unix::fs::{OpenOptionsExt, symlink}; + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt, symlink}; // Everything up to and including the rename can refuse, and only that // refuses: the link has not moved when `FlipRefused` is returned. @@ -823,6 +823,10 @@ impl StaticReleasePort for FilesystemStaticRelease { .create_new(true) .mode(0o444) .open(&temporary_carry)?; + // The creation mode is masked by the service's umask + // (UMask=0027 yields 0440); the sealed-tree verifier accepts + // exactly 0444, so say it on the open handle. + file.set_permissions(fs::Permissions::from_mode(0o444))?; file.write_all(&bytes)?; file.sync_all()?; fs::rename(&temporary_carry, &carried)?; @@ -17045,6 +17049,43 @@ processes 9 Ok(()) } + /// The service runs with UMask=0027, which masks the create mode to 0440 + /// and hides the carry from the web user. The umask is process-global, so + /// the proof runs in a child copy of this test binary and no sibling test + /// ever sees it. + #[cfg(unix)] + #[test] + fn static_release_carry_is_0444_under_the_service_umask() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + const CHILD: &str = "IDUNN_TEST_CARRY_UMASK_CHILD"; + const NAME: &str = "static_release_carry_is_0444_under_the_service_umask"; + if std::env::var_os(CHILD).is_none() { + let path = format!("{}::{NAME}", module_path!().split_once("::").map_or("", |(_, rest)| rest)); + let output = std::process::Command::new(std::env::current_exe()?) + .args(["--exact", path.as_str(), "--nocapture", "--test-threads=1"]) + .env(CHILD, "1") + .output()?; + assert!( + output.status.success(), + "child under umask 0027 failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("1 passed"), + "the child filter matched no test:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + return Ok(()); + } + unsafe { libc::umask(0o027) }; + let site = stand_in_site()?; + FilesystemStaticRelease.carry_and_flip(&site.binding, RELEASE_A)?; + let carried = site.binding.current_link.join("config.json"); + assert_eq!(fs::metadata(&carried)?.permissions().mode() & 0o777, 0o444); + Ok(()) + } + #[cfg(unix)] #[test] fn static_release_missing_site_config_fails_before_the_link_moves() -> Result<()> { From d13d98a08342d59ca43b8cc5c31ea6ba44a16026 Mon Sep 17 00:00:00 2001 From: Eureka Date: Sat, 14 Nov 2026 08:44:30 +0000 Subject: [PATCH 10/10] Carry comment: the web server reads the file, no verifier enforces it Co-Authored-By: Claude Sonnet 5.5 --- src/drivers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers.rs b/src/drivers.rs index 714c460..7dfeaee 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -824,8 +824,8 @@ impl StaticReleasePort for FilesystemStaticRelease { .mode(0o444) .open(&temporary_carry)?; // The creation mode is masked by the service's umask - // (UMask=0027 yields 0440); the sealed-tree verifier accepts - // exactly 0444, so say it on the open handle. + // (UMask=0027 yields 0440), and the web server reads this file + // as "other"; say 0444 on the open handle. file.set_permissions(fs::Permissions::from_mode(0o444))?; file.write_all(&bytes)?; file.sync_all()?;