diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index d1f52d2..7b2573b 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -390,6 +390,8 @@ read-only command on Yggdrasil on 2026-09-29. Nothing was written to the host. (`:3670`, `:3715`), and topology refresh (`:3780`). After three refused continuity restarts, supervision stops restarting (`:3522-3528`) but keeps supervising the route of a dead incarnation. Read. + Resolved by `cut-liveness-gates-presence`: route challenge and topology + refresh now run only for an incarnation observed running. - F9. The continuity restart limit counts refused restarts per generation (`is_refused_restart`, `:3469-3488`). A restart that succeeds admits a new generation and resets the count. A unit that starts, is admitted, and then diff --git a/src/control_plane.rs b/src/control_plane.rs index 34d1b87..83a266b 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5525,14 +5525,23 @@ fn render_supervision( } if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { lines.push(format!( - " route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}", - match route.degraded_since_unix_millis { + " route {} consecutive-failures {} last-challenge-at {}{} next-challenge-at {}{}", match route.degraded_since_unix_millis { Some(since) => format!("degraded-since {since}"), None => "healthy".to_owned(), }, route.consecutive_failures, at(route.last_challenge_at_unix_millis), - at(route.next_challenge_at_unix_millis) + route + .last_challenge_at_unix_millis + .map_or(String::new(), |last| format!(" ({} ms ago)", now.saturating_sub(last))), + at(route.next_challenge_at_unix_millis), + // Status sees the stored schedule, not why it is not being kept + // (a refused write, a transaction, a daemon that is down): it + // says only how overdue the challenge is. + route + .next_challenge_at_unix_millis + .filter(|next| *next < now) + .map_or(String::new(), |next| format!(" (past due {} ms)", now - next)) )); } if generation.is_none_or(|value| value.route_supervision.is_some()) @@ -5684,6 +5693,11 @@ struct Engine { /// One `ReportOnce` per faulting record, so one wedged transaction says /// its fault once and never drowns another's. fault_reports: Mutex>, + /// The key of every `report_once` line that reached stderr: what a test + /// counts, since stderr is not observable from inside it, and a key's + /// own entry is dropped when its fault clears. + #[cfg(test)] + printed_reports: Mutex>, /// Retry pacing for a transaction whose step keeps failing, by /// transaction id: consecutive failures and the earliest next attempt. /// Process-local on purpose: a restart retries at once, and no schema @@ -5811,6 +5825,8 @@ impl Engine { runner_stand_in: Mutex::default(), history_report: ReportOnce::default(), fault_reports: Mutex::default(), + #[cfg(test)] + printed_reports: Mutex::default(), resume_backoff: Mutex::default(), lift_held: Mutex::default(), lift_unsettled: std::sync::atomic::AtomicBool::new(false), @@ -5929,6 +5945,8 @@ impl Engine { .or_default() .offer(Some(message)); if let Some(message) = offered { + #[cfg(test)] + self.printed_reports.lock().unwrap().push(key.to_string()); eprintln!("{message}"); } } @@ -6730,10 +6748,23 @@ impl Engine { } fn clear_fault(&self, transaction_id: &str) { - self.fault_reports - .lock() - .expect("fault report mutex") - .remove(transaction_id); + let mut reports = self.fault_reports.lock().expect("fault report mutex"); + reports.remove(transaction_id); + reports.remove(&route_proof_report_key(transaction_id)); + } + + /// Say why the stable route's proof failed. The Routing phase retries the + /// proof until its deadline, and the deadline's own cause names only the + /// clock, so this line is where an operator learns what the listener + /// actually did. Said once per distinct reason while the fault lasts. + fn report_route_proof_failure(&self, transaction_id: &str, error: &anyhow::Error) { + self.report_once( + &route_proof_report_key(transaction_id), + format!( + "Idunn route proof for {transaction_id} failed: {}", + route_proof_reason(error) + ), + ); } /// Write `detail` as the record's `last_error`, touching nothing else and @@ -7253,37 +7284,18 @@ impl Engine { || snapshot.on_target(¤t.value.target).any(|stored| { stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity }); - match self.supervise_admitted_route(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn rejected admitted {} route continuity: {error:#}", - current.value.target - ), - } - match self.refresh_admitted_topology(&snapshot, current) { - Ok(true) => { - progressed = true; - continue; - } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved admitted {} after rejecting topology observation: {error:#}", - current.value.target - ), - } match self.retire_stale_incarnations(&snapshot, current) { Ok(true) => { progressed = true; continue; } - Ok(false) => {} - Err(error) => eprintln!( - "Idunn preserved a stale {} incarnation projection: {error:#}", - current.value.target + Ok(false) => self.clear_fault(&format!("stale-incarnation:{}", current.value.target)), + Err(error) => self.report_once( + &format!("stale-incarnation:{}", current.value.target), + format!( + "Idunn preserved a stale {} incarnation projection: {error:#}", + current.value.target + ), ), } // Restore the admitted Expected before anything is asked of the @@ -7312,11 +7324,16 @@ impl Engine { } Ok(()) })(); - if let Err(error) = repair { - eprintln!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target - ); + let repair_key = format!("projection-repair:{}", current.value.target); + match repair { + Ok(()) => self.clear_fault(&repair_key), + Err(error) => self.report_once( + &repair_key, + format!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ), + ), } let mut operational_error = None; let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { @@ -7327,6 +7344,7 @@ impl Engine { ) }) { Ok(observation) => { + self.clear_fault(&format!("observe:{}", current.value.target)); let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { let lease_health = (|| -> Result { let lease_path = current @@ -7347,6 +7365,10 @@ impl Engine { } bail!("write-lease store contains unexpected authority") })(); + let lease_key = format!("write-lease:{}", current.value.target); + if lease_health.is_ok() { + self.clear_fault(&lease_key); + } match lease_health { Ok(true) => false, Ok(false) => { @@ -7355,9 +7377,12 @@ impl Engine { true } Err(error) => { - eprintln!( - "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", - current.value.target + self.report_once( + &lease_key, + format!( + "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", + current.value.target + ), ); continue; } @@ -7377,13 +7402,20 @@ impl Engine { // silence, not a death; counting it as one burned every // continuity attempt in the reattach window on // 2026-09-11 and left a healthy Muninn demoted. - eprintln!( - "Idunn cannot observe admitted {} this tick: {error:#}", - current.value.target + // Said once while it lasts, and cleared the moment the host + // answers: a reattach window is not a line per tick. + self.report_once( + &format!("observe:{}", current.value.target), + format!( + "Idunn cannot observe admitted {} this tick: {error:#}", + current.value.target + ), ); continue; } Err(error) => { + // The host answered, with a death: no longer silence. + self.clear_fault(&format!("observe:{}", current.value.target)); operational_error = Some(error); None } @@ -7422,6 +7454,42 @@ impl Engine { ) }), ); + // Route and topology supervision are about a running incarnation; + // workload liveness decides whether there is one (route-continuity-cut F8, + // resolved by this order). Topology cannot bind without an observed + // activation, so a HostUnobservable tick skips both too. + let route_key = format!("route-supervision:{}", current.value.target); + match self.supervise_admitted_route(&snapshot, current) { + Ok(true) => { + self.clear_fault(&route_key); + progressed = true; + continue; + } + Ok(false) => self.clear_fault(&route_key), + Err(error) => self.report_once( + &route_key, + format!( + "Idunn rejected admitted {} route continuity: {error:#}", + current.value.target + ), + ), + } + let topology_key = format!("topology:{}", current.value.target); + match self.refresh_admitted_topology(&snapshot, current) { + Ok(true) => { + self.clear_fault(&topology_key); + progressed = true; + continue; + } + Ok(false) => self.clear_fault(&topology_key), + Err(error) => self.report_once( + &topology_key, + format!( + "Idunn preserved admitted {} after rejecting topology observation: {error:#}", + current.value.target + ), + ), + } continue; } let workload_error = operational_error @@ -7559,10 +7627,17 @@ impl Engine { // could not resolve. So a failed demotion mints nothing. The // failure is written to the generation, and the next attempt is // not before the deferral ends. + let demotion_key = format!("demotion:{}", current.value.target); + if demotion.is_ok() { + self.clear_fault(&demotion_key); + } if let Err(error) = demotion { - eprintln!( - "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", - current.value.target + self.report_once( + &demotion_key, + format!( + "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", + current.value.target + ), ); let mut deferred = supervision.clone(); deferred.defer_continuity(now, &bounded_detail(&format!("{error:#}"))); @@ -7689,17 +7764,13 @@ impl Engine { "admitted route receipt names another incarnation" ); let binding = current.value.plan.parsed_inputs()?.1; - let lease_driver = if let Some(lease) = current.value.leasing.lease() { + let lease_driver = if current.value.leasing.lease().is_some() { let lease_path = &binding .process_write_lease .as_ref() .context("stateful admitted generation has no write-lease binding")? .record_path; let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); - ensure!( - driver.observe_exact(lease)?, - "admitted process write lease is no longer exact" - ); Some(driver) } else { None @@ -8864,18 +8935,21 @@ impl Engine { rollback_allowed, &gate, )?; - let observation = match self.prove_stable_route( + let observation = match self.prove_routed_candidate_through_handoff( &admitted.value, &driver, membership_sha256, + preflight, ) { Ok(observation) => observation, Err(proof_error) if !rollback_allowed => { + self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return Err(proof_error).context( "incumbent authority was fenced; candidate route remains installed for fail-closed retry", ); } Err(proof_error) => { + self.report_route_proof_failure(&admitted.value.transaction_id, &proof_error); return match driver.rollback( expected, &activation.runtime_instance_id, @@ -9790,6 +9864,42 @@ impl Engine { ) } + /// Routing's proof, waiting out nginx's reload handoff. `systemctl reload` + /// only signals the master: the old workers keep answering the stable + /// listener until the new configuration is parsed and they are told to + /// quit, so Idunn's first challenge can reach the incumbent. An answer + /// that names the preflight's incumbent is therefore not-yet for + /// `ROUTE_RELOAD_HANDOFF`; it can delay and never admit (the id is read + /// unauthenticated, and the candidate's proof is checked as before). + /// Anything else fails the attempt at once. One challenge per poll and + /// nothing else: no install, rollback, route gate or persistence. + fn prove_routed_candidate_through_handoff( + &self, + transaction: &DeploymentTransaction, + driver: &NginxRouteDriver, + membership_sha256: String, + preflight: &crate::drivers::RoutePreflightReceipt, + ) -> Result { + let candidate = &required(&transaction.activation, "route activation")?.runtime_instance_id; + let until = handoff_now() + ROUTE_RELOAD_HANDOFF; + loop { + match self.prove_stable_route(transaction, driver, membership_sha256.clone()) { + Err(error) + if preflight.changes_route() + && handoff_now() < until + && error.downcast_ref::().is_some_and(|answer| { + preflight.incumbent_runtime_instance_id.as_deref() + == Some(answer.answered_instance.as_str()) + && answer.answered_instance != *candidate + }) => + { + handoff_sleep(ROUTE_RELOAD_HANDOFF_POLL); + } + result => return result, + } + } + } + fn prove_stable_route_against( &self, expected: &IdunnExpectedIncarnationRecord, @@ -9814,7 +9924,15 @@ impl Engine { challenged_at_unix_millis, received_at_unix_millis, &response.canonical_presence, - )?; + ) + .map_err(|error| match presence_answer_ids(&response.canonical_presence) { + Some((answered_instance, answered_activation_signer)) => error.context(RouteProofAnswer { + answered_instance: route_proof_label(&answered_instance), + answered_activation_signer: route_proof_label(&answered_activation_signer), + candidate_instance: route_proof_label(&activation.runtime_instance_id), + }), + None => error, + })?; let observation = RouteObservation { route_id: driver.binding.route_id.clone(), runtime_instance_id: activation.runtime_instance_id.clone(), @@ -10560,8 +10678,15 @@ impl Engine { .iter() .any(|request| request.value.command_id == transaction.command_id); let cause = if now >= deadline.deadline_at_unix_millis { + // The step's latest failure is the reason the phase never + // finished, and the abort that ends it must not drop it. + let last_attempt = transaction + .last_error + .as_deref() + .map(|error| format!("; its last attempt failed: {error}")) + .unwrap_or_default(); format!( - "the {:?} phase passed its deadline at unix ms {}", + "the {:?} phase passed its deadline at unix ms {}{last_attempt}", transaction.phase, deadline.deadline_at_unix_millis ) } else if requested { @@ -11426,6 +11551,109 @@ fn runner_failure_journal_line( } } +fn route_proof_report_key(transaction_id: &str) -> String { + format!("route-proof:{transaction_id}") +} + +/// How long Routing waits for nginx's old workers to let go of the stable +/// listener after a reload, derived from two ceilings: one Forward actuation +/// must suffice per attempt (12 an hour), and the 120 s Routing deadline must +/// keep room for at least four more backoff-spaced attempts if the route +/// genuinely never switches. +const ROUTE_RELOAD_HANDOFF: Duration = + if cfg!(test) { Duration::from_millis(400) } else { Duration::from_secs(10) }; +const ROUTE_RELOAD_HANDOFF_POLL: Duration = + if cfg!(test) { Duration::from_millis(20) } else { Duration::from_millis(250) }; + +/// The handoff loop's clock. Production reads the real one; a test may install +/// a virtual one on its own thread, so the window and the poll cadence are +/// measured exactly, whatever the host load. +#[cfg(not(test))] +fn handoff_now() -> std::time::Instant { + std::time::Instant::now() +} +#[cfg(not(test))] +fn handoff_sleep(pause: Duration) { + thread::sleep(pause); +} +#[cfg(test)] +thread_local! { + /// (virtual offset, sleeps taken, clock reads) while a test holds it. + static HANDOFF_CLOCK: std::cell::RefCell, usize)>> = + const { std::cell::RefCell::new(None) }; +} +#[cfg(test)] +fn handoff_now() -> std::time::Instant { + static BASE: std::sync::OnceLock = std::sync::OnceLock::new(); + let base = *BASE.get_or_init(std::time::Instant::now); + HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { + Some((offset, _, reads)) => { + *reads += 1; + assert!(*reads < 10_000, "the handoff loop spins without sleeping"); + base + *offset + } + None => std::time::Instant::now(), + }) +} +#[cfg(test)] +fn handoff_sleep(pause: Duration) { + HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { + Some((offset, sleeps, _)) => { + *offset += pause; + sleeps.push(pause); + } + None => thread::sleep(pause), + }) +} + +/// Who answered a stable route challenge that failed authentication, as the +/// answer claimed itself: ids only, unauthenticated, never signature bytes. +/// A label on a rejection, and the one thing that may say "challenge again". +#[derive(Debug)] +struct RouteProofAnswer { + answered_instance: String, + answered_activation_signer: String, + candidate_instance: String, +} + +impl std::fmt::Display for RouteProofAnswer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "stable route answered as runtime instance {}, activation signer {} (unauthenticated); the candidate is {}", + self.answered_instance, self.answered_activation_signer, self.candidate_instance + ) + } +} + +/// The ids a presence claims, if it decodes at all. No re-encode check and no +/// validation: this labels a rejection and is never evidence. +fn presence_answer_ids(canonical: &[u8]) -> Option<(String, String)> { + let presence = + rmp_serde::from_slice::(canonical).ok()?; + Some((presence.runtime_instance_id, presence.activation_signer_identity_id)) +} + +/// An id cut to 96 bytes at a char boundary, control characters stripped. +fn route_proof_label(id: &str) -> String { + let id: String = id.chars().filter(|c| !c.is_control()).collect(); + id[..id.floor_char_boundary(96)].to_owned() +} + +/// The stable route proof's failure in words: whether the listener said +/// nothing or answered wrongly, then the specific refusal (HTTP status line, +/// presence not active, challenge or authority disagreement). The texts it +/// carries name states and codes, never keys or signature bytes. A rejected +/// answer names the instance and activation signer it claimed, unauthenticated, +/// ids only. +fn route_proof_reason(error: &anyhow::Error) -> String { + match error.downcast_ref::() { + Some(ChallengeFailure::Silent(_)) => format!("silent, nothing answered: {error:#}"), + Some(ChallengeFailure::Refused(_)) => format!("refused: {error:#}"), + None => format!("{error:#}"), + } +} + fn bounded_detail(value: &str) -> String { let value = value.replace('', ""); if value.is_empty() { @@ -14129,6 +14357,10 @@ mod tests { .replace( "/var/lib/gamecult/idunn/service-deployment-brake.cc", &world.root.join("service-deployment-brake.cc").display().to_string(), + ) + .replace( + "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", + &world.root.join("service-lifecycle-brake.cc").display().to_string(), ), )?; let recipe = RECIPE @@ -15032,6 +15264,132 @@ mod tests { odin_reports_ready(world, transaction, sequence) } + /// An Odin-correlated generation, admitted over `workload`, with the + /// sequence of the reading it holds. + fn odin_admitted_world( + workload: Arc, + ) -> Result<(EngineFixture, DeploymentTransaction, u64)> { + let world = EngineFixture::with_workload(workload)?; + let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; + let first = seeded_sequence(None) + 1; + odin_reports_ready(&world, &seeded, first)?; + drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; + assert!(world.engine.retire_one_terminal_transaction()?); + Ok((world, seeded, first)) + } + + /// Odin's next reading, about an activation that is not the current one. + fn odin_republishes_another_activation( + world: &EngineFixture, + seeded: &DeploymentTransaction, + sequence: u64, + ) -> Result<()> { + let mut other = seeded.clone(); + other.activation.as_mut().context("no activation")?.runtime_instance_id = + sha256_id(b"another-incarnation"); + odin_republishes(world, &other, sequence) + } + + /// Engage the admitted target's lifecycle brake, at the path its seeded + /// binding names: a dead incarnation mints no continuity, so every later + /// tick reaches the same gates. + fn engage_lifecycle_brake_in(world: &EngineFixture) -> Result<()> { + let generation = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no admitted generation")? + .value + .clone(); + crate::provisioning::replace_lifecycle_brake( + &world.root.join("service-lifecycle-brake.cc"), + IdunnLifecycleBrakeRecord { + schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), + authority: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), + runtime_id: generation.expected.runtime_id.clone(), + target: "service".into(), + scope: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), + status: "engaged".into(), + reason: "test".into(), + updated_at_unix_millis: now_millis()?, + released_until_unix_millis: None, + }, + ) + } + + #[test] + fn a_dead_generation_does_not_take_odin_correlations() -> Result<()> { + let switch = SwitchWorkload::new(); + let (world, seeded, first) = odin_admitted_world(switch.clone())?; + switch.kill(); + // No continuity is minted, so no transaction ever owns the target and + // every tick reaches the topology gate. + engage_lifecycle_brake_in(&world)?; + let envelope = incumbent_envelope(&world)?; + + // A newer reading that binds the admitted activation, and one that does + // not: neither is taken, nor said, about an incarnation that is not there. + for reading in 0..2 { + if reading == 0 { + odin_republishes(&world, &seeded, first + 4)?; + } else { + odin_republishes_another_activation(&world, &seeded, first + 5)?; + } + for _ in 0..100 { + world.engine.supervise_one_admitted_generation()?; + } + assert!( + ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), + "a transaction owns the target: later ticks never reach the gate" + ); + assert_eq!(incumbent_envelope(&world)?, envelope); + assert!( + !world.engine.fault_reports.lock().unwrap().contains_key("topology:service"), + "a dead generation reported a topology rejection" + ); + } + Ok(()) + } + + fn printed(world: &EngineFixture, key: &str) -> usize { + world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() + } + + #[test] + fn a_running_generation_with_a_non_binding_correlation_is_reported_once() -> Result<()> { + let (world, seeded, first) = odin_admitted_world(Arc::new(StillWorkload))?; + odin_republishes_another_activation(&world, &seeded, first + 4)?; + let envelope = incumbent_envelope(&world)?; + for _ in 0..50 { + assert!(!world.engine.supervise_one_admitted_generation()?); + } + assert_eq!(incumbent_envelope(&world)?, envelope); + // The engine said it once over fifty ticks. + assert_eq!(printed(&world, "topology:service"), 1); + assert!(world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); + + // A binding reading is adopted and ends the fault, so the key clears. + odin_republishes(&world, &seeded, first + 6)?; + assert!(world.engine.supervise_one_admitted_generation()?); + assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); + let adopted = ControlSnapshot::read(&world.state_store)? + .admitted_for("service") + .context("no generation")? + .value + .clone(); + assert_eq!(adopted.odin_publisher_sequence_cursor, first + 6); + + // A recurrence is said again, once; a reading withdrawn ends it too. + odin_republishes_another_activation(&world, &seeded, first + 8)?; + for _ in 0..20 { + world.engine.supervise_one_admitted_generation()?; + } + assert_eq!(printed(&world, "topology:service"), 2); + let store = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); + assert!(store.delete_batch_if_unchanged(&store.pull_all_read_only_snapshot()?)?); + assert!(!world.engine.supervise_one_admitted_generation()?); + assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); + Ok(()) + } + fn ready_sequence(transaction: &DeploymentTransaction) -> Option { transaction .ready @@ -16954,6 +17312,60 @@ mod tests { Ok(()) } + #[test] + fn a_route_proof_failure_names_what_the_listener_did() { + let reason = |error: anyhow::Error| route_proof_reason(&error); + let silent = anyhow::Error::new(ChallengeFailure::Silent(anyhow!( + "connecting stable CultNet HTTP route 127.0.0.1:9: refused" + ))); + let text = reason(silent); + assert!(text.contains("silent") && text.contains("127.0.0.1:9"), "{text}"); + + let status = anyhow::Error::new(ChallengeFailure::Refused(anyhow!( + "stable CultNet HTTP route rejected the snapshot challenge: HTTP/1.1 502 Bad Gateway" + ))) + .context("candidate did not answer its stable route challenge"); + let text = reason(status); + assert!(text.contains("refused") && text.contains("502 Bad Gateway"), "{text}"); + + let inactive = anyhow!("route proof runtime is warming, not active"); + assert!(reason(inactive).contains("warming, not active")); + + let disagrees = anyhow::Error::new(PresenceDisagrees { + disagreements: vec![OdinTopologyDisagreement { + code: "runtime-instance-mismatch".into(), + expected: Some("a".into()), + observed: Some("b".into()), + }], + }); + let text = reason(disagrees); + assert!(text.contains("runtime-instance-mismatch (expected a, observed b)"), "{text}"); + + let unbound = anyhow!("route proof response is not bound to the exact challenge"); + assert!(reason(unbound).contains("not bound to the exact challenge")); + } + + #[test] + fn a_route_proof_failure_is_reported_once_per_reason_and_cleared_on_recovery() -> Result<()> { + let world = EngineFixture::new()?; + let key = route_proof_report_key("tx-1"); + let said = |world: &EngineFixture| { + world + .engine + .fault_reports + .lock() + .unwrap() + .get(&key) + .and_then(|report| report.last.lock().unwrap().clone()) + }; + world.engine.report_route_proof_failure("tx-1", &anyhow!("route proof runtime is warming, not active")); + let first = said(&world).context("no report")?; + assert!(first.contains("tx-1") && first.contains("warming, not active"), "{first}"); + world.engine.clear_fault("tx-1"); + assert!(said(&world).is_none(), "recovery left the proof report armed"); + Ok(()) + } + #[test] fn an_unretirable_legacy_marked_record_shows_its_fault_in_the_record() -> Result<()> { write_fixture_anchor()?; @@ -21566,6 +21978,25 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } + #[test] + fn status_says_how_overdue_a_route_challenge_is_and_claims_no_cause() -> Result<()> { + let now = 10_000_000_u64; + let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; + generation.route_supervision = Some(RouteSupervisionState { + last_challenge_at_unix_millis: Some(now - 5_000), + next_challenge_at_unix_millis: Some(now - 4_000), + ..RouteSupervisionState::default() + }); + let lines = render_supervision("service", Some(&generation), None, now).join("\n"); + assert!(lines.contains("last-challenge-at 9995000 (5000 ms ago)"), "{lines}"); + assert!(lines.contains("next-challenge-at 9996000 (past due 4000 ms)"), "{lines}"); + assert!(!lines.contains("waits on"), "{lines}"); + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(now); + let lines = render_supervision("service", Some(&generation), None, now).join("\n"); + assert!(!lines.contains("past due"), "{lines}"); + Ok(()) + } + #[test] fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { let now = 10_000_000_u64; @@ -21665,6 +22096,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation: IdunnRuntimeActivationRecord, provider: cultnet_rs::ServiceIdentitySigner, activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, + /// A second key the transaction's activation record never named. + other_signer: cultnet_rs::IdunnRuntimeActivationSigner, + /// The instance the stable listener answered as before the reload. + incumbent_instance: Mutex>, + stable_answers: AtomicUsize, state: Mutex<&'static str>, capacity: AtomicU32, sequence: AtomicU64, @@ -21694,11 +22130,36 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// A first Odin candidate answering its Warming challenge: the /// detail names the challenge as `idunn-warming:`. FirstOdinWarming, + /// The stable listener's first n answers come from the incumbent + /// (its instance id, a key the record never named), then Honest. + IncumbentFirst(usize), + /// The candidate's own instance id, signed with the wrong key. + ForeignActivation, + /// Neither the candidate nor the incumbent, wrong key. + StrangerInstance, } impl RuntimeStub { fn answer(&self, message_id: &str) -> Result> { + self.answer_on(message_id, false) + } + + fn answer_on(&self, message_id: &str, stable: bool) -> Result> { let expected = &self.expected; + let own = self.activation.runtime_instance_id.clone(); + let (instance, signer) = match *self.reply.lock().unwrap() { + Reply::IncumbentFirst(n) + if stable && self.stable_answers.fetch_add(1, Ordering::SeqCst) < n => + { + let incumbent = self.incumbent_instance.lock().unwrap().clone(); + (incumbent.context("no incumbent instance set")?, &self.other_signer) + } + Reply::ForeignActivation if stable => (own, &self.other_signer), + Reply::StrangerInstance if stable => { + (sha256_id(b"stranger instance"), &self.other_signer) + } + _ => (own, &self.activation_signer), + }; let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), target: expected.target.clone(), @@ -21710,7 +22171,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide state_schema_generation: expected.state_schema_generation.clone(), state_contract_sha256: expected.state_contract_sha256.clone(), runtime_id: expected.runtime_id.clone(), - runtime_instance_id: self.activation.runtime_instance_id.clone(), + runtime_instance_id: instance, bound_endpoint: expected .route .as_ref() @@ -21737,7 +22198,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide observed_at_unix_millis: now_millis()?, signature_algorithm: "ed25519".into(), signature: Vec::new(), - activation_signer_identity_id: self.activation_signer.identity_id(), + activation_signer_identity_id: signer.identity_id(), activation_signature: Vec::new(), }; let proof = presence.canonical_proof_payload()?; @@ -21745,8 +22206,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .provider .sign::(&proof) .signature; - presence.activation_signature = - self.activation_signer.sign_presence_proof(&presence)?; + presence.activation_signature = signer.sign_presence_proof(&presence)?; Ok(rmp_serde::to_vec(&presence)?) } @@ -21754,7 +22214,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide *self.state.lock().unwrap() = state; } - fn serve_one(&self, mut stream: TcpStream) -> Result<()> { + fn serve_one(&self, mut stream: TcpStream, stable: bool) -> Result<()> { stream.set_read_timeout(Some(Duration::from_secs(3)))?; let mut request = Vec::new(); let mut chunk = [0_u8; 4096]; @@ -21801,7 +22261,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide record_key: self.expected.target.clone(), stored_at: rfc3339_millis(now_millis()?)?, payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, - payload: self.answer(&message_id)?, + payload: self.answer_on(&message_id, stable)?, source_runtime_id: None, source_agent_id: None, source_role: None, @@ -21843,7 +22303,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide hook(); } if !stub.hang_up.load(Ordering::SeqCst) { - let _ = stub.serve_one(stream); + let _ = stub.serve_one(stream, !candidate); } } Err(_) => std::thread::sleep(Duration::from_millis(5)), @@ -21955,6 +22415,10 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .replace( "/etc/nginx/idunn-stream-routes/service.conf", &config.display().to_string().replace('\\', "/"), + ) + .replace( + "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", + &world.root.join("service-lifecycle-brake.cc").display().to_string(), ); let (binding, recipe_source) = if stateful { std::fs::create_dir_all(world.root.join("lease"))?; @@ -22059,6 +22523,14 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide )?; let mut seed = Vec::new(); let activation = launch.write_credential(&mut seed)?; + let mut other_seed = Vec::new(); + IdunnRuntimeActivationLaunch::issue( + &expected, + sha256_id(b"unrecorded activation"), + now, + &world.engine.idunn_signer, + )? + .write_credential(&mut other_seed)?; let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; let mut workload = recorded.workload.clone().context("no recorded workload")?; match &mut workload { @@ -22090,6 +22562,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( &seed[..], )?, + other_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( + &other_seed[..], + )?, + incumbent_instance: Mutex::new(None), + stable_answers: AtomicUsize::new(0), state: Mutex::new("warming"), capacity: AtomicU32::new(minimum_capacity), sequence: AtomicU64::new(0), @@ -24069,30 +24546,325 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[cfg(unix)] #[test] - fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { + fn a_dead_unit_is_restarted_without_a_route_challenge() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.admit()?; routed.workload.kill(); routed.stub.hang_up.store(true, Ordering::SeqCst); routed.drift()?; + let before = admitted(&routed)?.route_supervision; + let reloads = routed.reloads(); + let survivals = meters_of(&routed.world, "service")?.route_survivals; - // Tick 1: the repair charges the target's meters and the proof - // fails. Both are writes, so they end the pass: nothing else is - // decided from the snapshot they made stale. - assert!(routed.tick()?); - let state = admitted(&routed)?.route_supervision.context("no route state")?; - assert_eq!(state.consecutive_failures, 1); - assert!(state.degraded_since_unix_millis.is_some()); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - let meters = meters_of(&routed.world, "service")?; - assert_eq!(meters.route_survivals.len(), 2); - assert!(meters.route_actuations.is_empty()); - - // Tick 2: the route waits, and the dead unit is restarted from a - // fresh read, counted in the same write that mints it. + // Tick 1: restart, not route repair, is the response to a dead + // incarnation. The continuity is minted and no route survival is + // charged for a proof nobody could have passed. assert!(routed.tick()?); assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); + assert_eq!(meters_of(&routed.world, "service")?.route_survivals, survivals); + assert_eq!(admitted(&routed)?.route_supervision, before); + assert_eq!(routed.reloads(), reloads, "the dead unit's route was repaired"); + Ok(()) + } + + /// A target with a process write lease, admitted and running. + #[cfg(unix)] + fn admitted_stateful_world() -> Result { + let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; + routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; + let granted_lease = routed + .transaction()? + .leasing + .as_ref() + .and_then(LeasingEvidence::lease_sha256) + .context("no granted lease")? + .to_owned(); + routed.stub.set_state("active"); + *routed.stub.lease.lock().unwrap() = Some(granted_lease); + routed.step()?; + routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; + routed.promote()?; + routed.drive_until(|transaction| transaction.is_terminal())?; + assert!(routed.world.engine.retire_one_terminal_transaction()?); + Ok(routed) + } + + /// Engage the target's lifecycle brake: the dead path may not mint. + fn engage_lifecycle_brake(routed: &RoutedWorld) -> Result<()> { + engage_lifecycle_brake_in(&routed.world) + } + #[cfg(unix)] + #[test] + fn a_parked_routed_target_is_not_route_supervised() -> Result<()> { + let routed = admitted_stateful_world()?; + let generation = admitted(&routed)?; + let lease = generation.leasing.lease().context("no lease")?.clone(); + CultCacheWriteLeaseDriver::new( + "service", + routed.world.root.join("lease/process-write-lease.cc"), + ) + .revoke_exact(Some(&lease))?; + routed.workload.kill(); + engage_lifecycle_brake(&routed)?; + routed.drift()?; + let envelope = incumbent_envelope(&routed.world)?; + let reloads = routed.reloads(); + let programs = routed.world.route_stubs.count(""); + + for _ in 0..100 { + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "a parked target's route was reloaded"); + assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); + assert_eq!(incumbent_envelope(&routed.world)?, envelope); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + let reports = routed.world.engine.fault_reports.lock().unwrap(); + assert!( + !reports.keys().any(|key| key.starts_with("route-supervision:") || key.starts_with("route:")), + "{:?}", + reports.keys().collect::>() + ); + Ok(()) + } + + fn fault_keys(routed: &RoutedWorld) -> Vec { + routed.world.engine.fault_reports.lock().unwrap().keys().cloned().collect() + } + + /// Soul's probe: a route record that cannot be written leaves the + /// challenge past due on a running incarnation. Status shows the age + /// and does not name a cause it cannot see. + #[cfg(unix)] + #[test] + fn status_of_a_past_due_route_on_a_running_incarnation_claims_no_wait() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.stub.hang_up.store(true, Ordering::SeqCst); + routed.drift()?; + assert!(routed.tick()?); + assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); + routed.stub.hang_up.store(false, Ordering::SeqCst); + edit_incumbent(&routed.world, |generation| { + generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); + })?; + let fragment = routed.fragment_path(); + std::fs::remove_file(&fragment)?; + std::fs::create_dir(&fragment)?; + for _ in 0..20 { + routed.tick()?; + } + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + assert!(fault_keys(&routed).contains(&"route-supervision:service".to_owned())); + let generation = admitted(&routed)?; + let lines = render_supervision("service", Some(&generation), None, now_millis()?).join("\n"); + assert!(lines.contains("(past due "), "{lines}"); + assert!(!lines.contains("waits on"), "{lines}"); + Ok(()) + } + + /// A target whose lease store is empty while its process runs is a + /// dead-path case: the route is not challenged for it. + #[cfg(unix)] + #[test] + fn a_running_target_with_an_empty_lease_is_not_route_challenged() -> Result<()> { + let routed = admitted_stateful_world()?; + let lease = admitted(&routed)?.leasing.lease().context("no lease")?.clone(); + CultCacheWriteLeaseDriver::new( + "service", + routed.world.root.join("lease/process-write-lease.cc"), + ) + .revoke_exact(Some(&lease))?; + engage_lifecycle_brake(&routed)?; + routed.drift()?; + let envelope = incumbent_envelope(&routed.world)?; + let reloads = routed.reloads(); + let programs = routed.world.route_stubs.count(""); + for _ in 0..50 { + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "an empty-lease target's route was reloaded"); + assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); + assert_eq!(incumbent_envelope(&routed.world)?, envelope); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + assert!( + !fault_keys(&routed).iter().any(|key| key.starts_with("route")), + "{:?}", + fault_keys(&routed) + ); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_host_that_answers_with_a_death_ends_the_silence_fault() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + engage_lifecycle_brake(&routed)?; + routed.workload.silence(true); + for _ in 0..5 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, "observe:service"), 1); + routed.workload.silence(false); + routed.workload.kill(); + routed.tick()?; + assert!(!fault_keys(&routed).contains(&"observe:service".to_owned())); + Ok(()) + } + + /// Replace the topology store with a directory: every read of it fails. + #[cfg(unix)] + fn break_topology(routed: &RoutedWorld) -> Result<()> { + let store = routed.world.root.join("topology.cc"); + if store.exists() { + std::fs::rename(&store, routed.world.root.join("topology.aside"))?; + } + std::fs::create_dir(&store)?; + Ok(()) + } + + #[cfg(unix)] + fn mend_topology(routed: &RoutedWorld) -> Result<()> { + let store = routed.world.root.join("topology.cc"); + std::fs::remove_dir(&store)?; + let aside = routed.world.root.join("topology.aside"); + if aside.exists() { + std::fs::rename(aside, &store)?; + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_stale_incarnation_and_a_projection_repair_that_fail_are_each_said_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let keys = ["stale-incarnation:service", "projection-repair:service"]; + for round in 1..=2 { + break_topology(&routed)?; + for _ in 0..30 { + routed.tick()?; + } + for key in keys { + assert_eq!(printed_in(&routed, key), round, "{key}"); + assert!(fault_keys(&routed).contains(&key.to_owned()), "{key}"); + } + mend_topology(&routed)?; + routed.tick()?; + for key in keys { + assert!(!fault_keys(&routed).contains(&key.to_owned()), "{key}"); + } + } + Ok(()) + } + + #[cfg(unix)] + #[test] + fn a_refused_demotion_is_said_once_and_cleared_when_it_succeeds() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.workload.kill(); + break_topology(&routed)?; + // Each attempt is let through by forgetting the deferral it wrote. + for _ in 0..6 { + set_meters(&routed.world, &TargetSupervision::new("service"))?; + routed.tick()?; + } + assert_eq!(printed_in(&routed, "demotion:service"), 1); + assert!(fault_keys(&routed).contains(&"demotion:service".to_owned())); + mend_topology(&routed)?; + set_meters(&routed.world, &TargetSupervision::new("service"))?; + routed.tick()?; + assert!(!fault_keys(&routed).contains(&"demotion:service".to_owned())); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn an_unsafe_write_lease_store_is_said_once_and_cleared_when_it_is_exact() -> Result<()> { + let routed = admitted_stateful_world()?; + let store = routed.world.root.join("lease/process-write-lease.cc"); + let exact = std::fs::read(&store)?; + for round in 1..=2 { + std::fs::write(&store, b"not a lease store")?; + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, "write-lease:service"), round); + assert!(fault_keys(&routed).contains(&"write-lease:service".to_owned())); + std::fs::write(&store, &exact)?; + routed.tick()?; + assert!(!fault_keys(&routed).contains(&"write-lease:service".to_owned())); + } + Ok(()) + } + + fn printed_in(routed: &RoutedWorld, key: &str) -> usize { + routed.world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() + } + + #[cfg(unix)] + #[test] + fn a_running_route_whose_fragment_cannot_be_read_is_reported_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + let key = "route-supervision:service"; + let fragment = routed.fragment_path(); + let bytes = std::fs::read(&fragment)?; + // The fragment is a directory: reading it fails, and nothing is + // written, so every tick meets the same error. + std::fs::remove_file(&fragment)?; + std::fs::create_dir(&fragment)?; + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, key), 1); + assert!(routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); + + // The fragment is back: the fault ends and the key clears. + std::fs::remove_dir(&fragment)?; + std::fs::write(&fragment, &bytes)?; + routed.tick()?; + assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); + + // A recurrence is said again, once. + std::fs::remove_file(&fragment)?; + std::fs::create_dir(&fragment)?; + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, key), 2); + Ok(()) + } + + /// Skipping route and topology on a tick the host could not be asked is + /// the steward's ruling (no activation can be bound without an + /// observation; no routed target runs on a host actuator today). + #[cfg(unix)] + #[test] + fn a_host_that_cannot_be_asked_skips_route_supervision_and_is_said_once() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + routed.admit()?; + routed.drift()?; + let envelope = incumbent_envelope(&routed.world)?; + let reloads = routed.reloads(); + routed.workload.silence(true); + for _ in 0..30 { + routed.tick()?; + } + assert_eq!(routed.reloads(), reloads, "route supervision ran without an observation"); + assert_eq!(incumbent_envelope(&routed.world)?, envelope); + assert_eq!(printed_in(&routed, "observe:service"), 1); + + routed.workload.silence(false); + routed.tick()?; + assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key("observe:service")); + routed.workload.silence(true); + for _ in 0..5 { + routed.tick()?; + } + assert_eq!(printed_in(&routed, "observe:service"), 2); Ok(()) } @@ -24257,9 +25029,39 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide routed.run_to_routing()?; let deadline = routed.transaction()?.phase_deadline.context("Routing has no deadline")?; routed.stub.hang_up.store(true, Ordering::SeqCst); - for _ in 0..3 { + // The Routing tick itself reports why the proof failed, once per + // distinct reason, and recovery clears it. + let proof_key = route_proof_report_key(&routed.transaction()?.transaction_id); + let said = |routed: &RoutedWorld| -> Option { + routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .get(&proof_key) + .and_then(|report| report.last.lock().unwrap().clone()) + }; + assert!(said(&routed).is_none(), "a proof report existed before any failure"); + routed.resume()?; + let first = said(&routed).context("the failing Routing tick reported no proof failure")?; + assert!(first.contains("route proof") && first.contains("silent, nothing answered"), "{first}"); + // The hung-up stub fails the same way each attempt, but the OS + // words the tail either as an early EOF or a broken pipe, so a + // retry may honestly report a second distinct reason. What may + // not change is everything up to that tail, and a repeat must not + // mint a new line each attempt. + let head = |text: &str| text.split("answered:").next().unwrap().to_owned(); + let mut seen = vec![first.clone()]; + for _ in 0..2 { routed.resume()?; + let again = said(&routed).context("a retry cleared the proof report")?; + assert_eq!(head(&again), head(&first), "a retry reworded the report"); + if !seen.contains(&again) { + seen.push(again); + } } + assert!(seen.len() <= 2, "the same failure was reported afresh on every retry: {seen:#?}"); // Before the deadline each attempt installs and rolls back, and no // resume moves the deadline. assert_eq!(routed.reloads(), 6); @@ -24302,6 +25104,200 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide assert_eq!(records.len(), 1); assert_eq!(records[0].close_reason, Some(CloseReason::Recovered)); assert_eq!(routed.workload.stops.load(std::sync::atomic::Ordering::SeqCst), stops); + assert!(said(&routed).is_none(), "recovery left the proof report armed"); + Ok(()) + } + + // ----------------------------------------------------------------- + // The reload handoff: nginx's old workers may answer the stable + // listener for a moment after `systemctl reload` returns. + // ----------------------------------------------------------------- + + /// A deploy world at Routing whose preflight found an incumbent route + /// (another instance, another membership), so the install changes it. + #[cfg(unix)] + fn handoff_world(reply: Reply) -> Result<(RoutedWorld, String)> { + handoff_world_with(reply, false) + } + + /// `same_membership`: the incumbent's membership equals the + /// candidate's, so the install changes no route. + #[cfg(unix)] + fn handoff_world_with(reply: Reply, same_membership: bool) -> Result<(RoutedWorld, String)> { + let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; + let incumbent = sha256_id(b"incumbent instance"); + let current = resident(&routed.world)?; + let mut next = current.value.clone(); + let mut on_disk = b"incumbent route\n".to_vec(); + if same_membership { + on_disk = NginxRouteDriver::new( + next.plan.as_ref().unwrap().parsed_inputs()?.1.route.context("no route binding")?, + ) + .render(next.expected.as_ref().context("no Expected")?)?; + } + let preflight = next.route_preflight.as_mut().context("no preflight")?; + preflight.incumbent_runtime_instance_id = Some(incumbent.clone()); + preflight.incumbent_membership_sha256 = Some(sha256_id(&on_disk)); + preflight.incumbent_configuration = Some(on_disk.clone()); + replace_transaction(&routed.world.state_store, ¤t, &next)?; + std::fs::write(routed.fragment_path(), &on_disk)?; + routed.stub.set_state("active"); + routed.run_to_routing()?; + *routed.stub.incumbent_instance.lock().unwrap() = Some(incumbent.clone()); + *routed.stub.reply.lock().unwrap() = reply; + Ok((routed, incumbent)) + } + + fn forwards(routed: &RoutedWorld) -> Result { + Ok(meters_of(&routed.world, "service")?.route_actuations.len()) + } + + fn proof_report(routed: &RoutedWorld) -> Result> { + let key = route_proof_report_key(&routed.transaction()?.transaction_id); + Ok(routed + .world + .engine + .fault_reports + .lock() + .unwrap() + .get(&key) + .and_then(|report| report.last.lock().unwrap().clone())) + } + + /// Holds a virtual handoff clock on this thread: the wait is measured + /// in the loop's own sleeps, so no host load can move it. + struct VirtualHandoffClock; + + impl VirtualHandoffClock { + fn hold() -> Self { + HANDOFF_CLOCK.with(|c| *c.borrow_mut() = Some((Duration::ZERO, Vec::new(), 0))); + Self + } + + /// The pauses the loop took so far. + fn sleeps(&self) -> Vec { + HANDOFF_CLOCK.with(|c| c.borrow().as_ref().unwrap().1.clone()) + } + } + + impl Drop for VirtualHandoffClock { + fn drop(&mut self) { + HANDOFF_CLOCK.with(|c| *c.borrow_mut() = None); + } + } + + #[cfg(unix)] + #[test] + fn routing_waits_out_the_incumbent_answering_after_reload() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, _) = handoff_world(Reply::IncumbentFirst(3))?; + let before = routed.reloads(); + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + routed.promote()?; + assert_eq!(routed.reloads() - before, 1, "an install and no rollback"); + assert_eq!(forwards(&routed)?, 1); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 4); + assert_eq!(clock.sleeps(), vec![ROUTE_RELOAD_HANDOFF_POLL; 3]); + assert!(proof_report(&routed)?.is_none(), "a route proof report remains"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn routing_fails_once_when_the_incumbent_never_lets_go() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, incumbent) = handoff_world(Reply::IncumbentFirst(usize::MAX))?; + let before = routed.reloads(); + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + let error = routed.step().unwrap_err(); + // The window is exactly ROUTE_RELOAD_HANDOFF of polls at exactly + // the poll cadence: one challenge before each pause and one after + // the last. + let sleeps = clock.sleeps(); + let polls = (ROUTE_RELOAD_HANDOFF.as_millis() / ROUTE_RELOAD_HANDOFF_POLL.as_millis()) as usize; + assert_eq!(sleeps, vec![ROUTE_RELOAD_HANDOFF_POLL; polls]); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, polls + 1); + assert_eq!(routed.reloads() - before, 2, "install and one rollback"); + assert_eq!(forwards(&routed)?, 1); + let said = proof_report(&routed)?.context("no route proof report")?; + let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; + let signer = routed.stub.other_signer.identity_id(); + for text in [format!("{error:#}"), said] { + assert!(text.contains(&format!("runtime instance {incumbent}")), "{text}"); + assert!(text.contains(&format!("activation signer {signer}")), "{text}"); + assert!(text.contains(&format!("the candidate is {candidate}")), "{text}"); + } + routed.step().unwrap_err(); + assert_eq!(forwards(&routed)?, 2, "the second attempt charges one more Forward"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn routing_fails_at_once_when_the_candidate_answers_with_a_foreign_activation() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, _) = handoff_world(Reply::ForeignActivation)?; + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + let error = routed.step().unwrap_err(); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert!(clock.sleeps().is_empty()); + let text = format!("{error:#}"); + let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; + assert!(text.contains("runtime activation proof verification failed"), "{text}"); + assert!(text.contains(&format!("runtime instance {candidate}")), "{text}"); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn routing_fails_at_once_for_a_stranger_instance() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, _) = handoff_world(Reply::StrangerInstance)?; + let before = routed.reloads(); + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + routed.step().unwrap_err(); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert!(clock.sleeps().is_empty()); + assert_eq!(routed.reloads() - before, 2); + Ok(()) + } + + /// An incumbent whose membership equals the candidate's has no route + /// to hand off, so its answer is a plain failure, never waited on. + #[cfg(unix)] + #[test] + fn routing_does_not_wait_when_the_route_is_unchanged() -> Result<()> { + let clock = VirtualHandoffClock::hold(); + let (routed, _) = handoff_world_with(Reply::IncumbentFirst(1), true)?; + let hits = routed.stub.stable_hits.load(Ordering::SeqCst); + routed.step().unwrap_err(); + assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); + assert!(clock.sleeps().is_empty()); + Ok(()) + } + + #[test] + fn route_proof_answer_names_ids_only() -> Result<()> { + let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; + let mut presence = rmp_serde::from_slice::( + &routed.stub.answer("probe")?, + )?; + presence.runtime_instance_id = format!("a\u{7}{}", "é".repeat(5_000)); + presence.activation_signer_identity_id = "b".repeat(10_240); + presence.signature = vec![0xAB; 64]; + presence.activation_signature = vec![0xCD; 64]; + let (instance, signer) = presence_answer_ids(&rmp_serde::to_vec(&presence)?) + .context("a decodable presence gave no ids")?; + let text = RouteProofAnswer { + answered_instance: route_proof_label(&instance), + answered_activation_signer: route_proof_label(&signer), + candidate_instance: route_proof_label(&"c".repeat(10_240)), + } + .to_string(); + assert!(text.len() <= 400, "{} bytes", text.len()); + assert!(!text.contains('\u{7}')); + assert!(!text.contains("abab") && !text.contains("cdcd")); + assert!(presence_answer_ids(b"not msgpack presence").is_none()); Ok(()) } @@ -24910,6 +25906,16 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .post_fencing_abort .context("the passed deadline did not end a deployment")?; assert!(abort.error.contains("Routing phase passed its deadline"), "{}", abort.error); + assert!( + abort.error.contains("its last attempt failed"), + "the deadline abort dropped the proof's failure: {}", + abort.error + ); + assert!( + abort.error.contains("stable CultNet route stopped before its headers ended"), + "the deadline abort dropped the proof's own failure text: {}", + abort.error + ); assert!( routed.world.engine.return_wedged_continuity_aborts_forward()?.is_empty(), "the boot lift took a deployment's abort" diff --git a/src/drivers.rs b/src/drivers.rs index 46cb916..7dfeaee 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -762,7 +762,7 @@ impl StaticReleasePort for FilesystemStaticRelease { binding: &StaticReleaseBinding, release_id: &str, ) -> Result { - use std::os::unix::fs::{OpenOptionsExt, symlink}; + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt, symlink}; // Everything up to and including the rename can refuse, and only that // refuses: the link has not moved when `FlipRefused` is returned. @@ -823,6 +823,10 @@ impl StaticReleasePort for FilesystemStaticRelease { .create_new(true) .mode(0o444) .open(&temporary_carry)?; + // The creation mode is masked by the service's umask + // (UMask=0027 yields 0440), and the web server reads this file + // as "other"; say 0444 on the open handle. + file.set_permissions(fs::Permissions::from_mode(0o444))?; file.write_all(&bytes)?; file.sync_all()?; fs::rename(&temporary_carry, &carried)?; @@ -17045,6 +17049,43 @@ processes 9 Ok(()) } + /// The service runs with UMask=0027, which masks the create mode to 0440 + /// and hides the carry from the web user. The umask is process-global, so + /// the proof runs in a child copy of this test binary and no sibling test + /// ever sees it. + #[cfg(unix)] + #[test] + fn static_release_carry_is_0444_under_the_service_umask() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + const CHILD: &str = "IDUNN_TEST_CARRY_UMASK_CHILD"; + const NAME: &str = "static_release_carry_is_0444_under_the_service_umask"; + if std::env::var_os(CHILD).is_none() { + let path = format!("{}::{NAME}", module_path!().split_once("::").map_or("", |(_, rest)| rest)); + let output = std::process::Command::new(std::env::current_exe()?) + .args(["--exact", path.as_str(), "--nocapture", "--test-threads=1"]) + .env(CHILD, "1") + .output()?; + assert!( + output.status.success(), + "child under umask 0027 failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("1 passed"), + "the child filter matched no test:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + return Ok(()); + } + unsafe { libc::umask(0o027) }; + let site = stand_in_site()?; + FilesystemStaticRelease.carry_and_flip(&site.binding, RELEASE_A)?; + let carried = site.binding.current_link.join("config.json"); + assert_eq!(fs::metadata(&carried)?.permissions().mode() & 0o777, 0o444); + Ok(()) + } + #[cfg(unix)] #[test] fn static_release_missing_site_config_fails_before_the_link_moves() -> Result<()> { diff --git a/src/provisioning.rs b/src/provisioning.rs index 5af3903..324785e 100644 --- a/src/provisioning.rs +++ b/src/provisioning.rs @@ -304,7 +304,7 @@ fn lifecycle_brake_status(options: &BTreeMap) -> Result<()> { Ok(()) } -fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { +pub(crate) fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { record.validate()?; let store = SingleFileMessagePackBackingStore::new(path); let entries = store.pull_all_read_only_snapshot()?;