diff --git a/docs/route-continuity-cut.md b/docs/route-continuity-cut.md index 7b2573b..d1f52d2 100644 --- a/docs/route-continuity-cut.md +++ b/docs/route-continuity-cut.md @@ -390,8 +390,6 @@ read-only command on Yggdrasil on 2026-09-29. Nothing was written to the host. (`:3670`, `:3715`), and topology refresh (`:3780`). After three refused continuity restarts, supervision stops restarting (`:3522-3528`) but keeps supervising the route of a dead incarnation. Read. - Resolved by `cut-liveness-gates-presence`: route challenge and topology - refresh now run only for an incarnation observed running. - F9. The continuity restart limit counts refused restarts per generation (`is_refused_restart`, `:3469-3488`). A restart that succeeds admits a new generation and resets the count. A unit that starts, is admitted, and then diff --git a/src/control_plane.rs b/src/control_plane.rs index 83a266b..11db393 100644 --- a/src/control_plane.rs +++ b/src/control_plane.rs @@ -5525,23 +5525,14 @@ fn render_supervision( } if let Some(route) = generation.and_then(|value| value.route_supervision.as_ref()) { lines.push(format!( - " route {} consecutive-failures {} last-challenge-at {}{} next-challenge-at {}{}", match route.degraded_since_unix_millis { + " route {} consecutive-failures {} last-challenge-at {} next-challenge-at {}", + match route.degraded_since_unix_millis { Some(since) => format!("degraded-since {since}"), None => "healthy".to_owned(), }, route.consecutive_failures, at(route.last_challenge_at_unix_millis), - route - .last_challenge_at_unix_millis - .map_or(String::new(), |last| format!(" ({} ms ago)", now.saturating_sub(last))), - at(route.next_challenge_at_unix_millis), - // Status sees the stored schedule, not why it is not being kept - // (a refused write, a transaction, a daemon that is down): it - // says only how overdue the challenge is. - route - .next_challenge_at_unix_millis - .filter(|next| *next < now) - .map_or(String::new(), |next| format!(" (past due {} ms)", now - next)) + at(route.next_challenge_at_unix_millis) )); } if generation.is_none_or(|value| value.route_supervision.is_some()) @@ -5693,11 +5684,6 @@ struct Engine { /// One `ReportOnce` per faulting record, so one wedged transaction says /// its fault once and never drowns another's. fault_reports: Mutex>, - /// The key of every `report_once` line that reached stderr: what a test - /// counts, since stderr is not observable from inside it, and a key's - /// own entry is dropped when its fault clears. - #[cfg(test)] - printed_reports: Mutex>, /// Retry pacing for a transaction whose step keeps failing, by /// transaction id: consecutive failures and the earliest next attempt. /// Process-local on purpose: a restart retries at once, and no schema @@ -5825,8 +5811,6 @@ impl Engine { runner_stand_in: Mutex::default(), history_report: ReportOnce::default(), fault_reports: Mutex::default(), - #[cfg(test)] - printed_reports: Mutex::default(), resume_backoff: Mutex::default(), lift_held: Mutex::default(), lift_unsettled: std::sync::atomic::AtomicBool::new(false), @@ -5945,8 +5929,6 @@ impl Engine { .or_default() .offer(Some(message)); if let Some(message) = offered { - #[cfg(test)] - self.printed_reports.lock().unwrap().push(key.to_string()); eprintln!("{message}"); } } @@ -7284,18 +7266,37 @@ impl Engine { || snapshot.on_target(¤t.value.target).any(|stored| { stored.value.is_live() && stored.value.command_kind == CommandKind::Continuity }); + match self.supervise_admitted_route(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn rejected admitted {} route continuity: {error:#}", + current.value.target + ), + } + match self.refresh_admitted_topology(&snapshot, current) { + Ok(true) => { + progressed = true; + continue; + } + Ok(false) => {} + Err(error) => eprintln!( + "Idunn preserved admitted {} after rejecting topology observation: {error:#}", + current.value.target + ), + } match self.retire_stale_incarnations(&snapshot, current) { Ok(true) => { progressed = true; continue; } - Ok(false) => self.clear_fault(&format!("stale-incarnation:{}", current.value.target)), - Err(error) => self.report_once( - &format!("stale-incarnation:{}", current.value.target), - format!( - "Idunn preserved a stale {} incarnation projection: {error:#}", - current.value.target - ), + Ok(false) => {} + Err(error) => eprintln!( + "Idunn preserved a stale {} incarnation projection: {error:#}", + current.value.target ), } // Restore the admitted Expected before anything is asked of the @@ -7324,16 +7325,11 @@ impl Engine { } Ok(()) })(); - let repair_key = format!("projection-repair:{}", current.value.target); - match repair { - Ok(()) => self.clear_fault(&repair_key), - Err(error) => self.report_once( - &repair_key, - format!( - "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", - current.value.target - ), - ), + if let Err(error) = repair { + eprintln!( + "Idunn preserved admitted {} after refusing topology projection repair: {error:#}", + current.value.target + ); } let mut operational_error = None; let observation = match self.workload_for(¤t.value.plan).and_then(|workload| { @@ -7344,7 +7340,6 @@ impl Engine { ) }) { Ok(observation) => { - self.clear_fault(&format!("observe:{}", current.value.target)); let lease_is_missing = if let Some(lease) = current.value.leasing.lease() { let lease_health = (|| -> Result { let lease_path = current @@ -7365,10 +7360,6 @@ impl Engine { } bail!("write-lease store contains unexpected authority") })(); - let lease_key = format!("write-lease:{}", current.value.target); - if lease_health.is_ok() { - self.clear_fault(&lease_key); - } match lease_health { Ok(true) => false, Ok(false) => { @@ -7377,12 +7368,9 @@ impl Engine { true } Err(error) => { - self.report_once( - &lease_key, - format!( - "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", - current.value.target - ), + eprintln!( + "Idunn preserved admitted {} after refusing an unsafe write-lease mutation: {error:#}", + current.value.target ); continue; } @@ -7402,20 +7390,13 @@ impl Engine { // silence, not a death; counting it as one burned every // continuity attempt in the reattach window on // 2026-09-11 and left a healthy Muninn demoted. - // Said once while it lasts, and cleared the moment the host - // answers: a reattach window is not a line per tick. - self.report_once( - &format!("observe:{}", current.value.target), - format!( - "Idunn cannot observe admitted {} this tick: {error:#}", - current.value.target - ), + eprintln!( + "Idunn cannot observe admitted {} this tick: {error:#}", + current.value.target ); continue; } Err(error) => { - // The host answered, with a death: no longer silence. - self.clear_fault(&format!("observe:{}", current.value.target)); operational_error = Some(error); None } @@ -7454,42 +7435,6 @@ impl Engine { ) }), ); - // Route and topology supervision are about a running incarnation; - // workload liveness decides whether there is one (route-continuity-cut F8, - // resolved by this order). Topology cannot bind without an observed - // activation, so a HostUnobservable tick skips both too. - let route_key = format!("route-supervision:{}", current.value.target); - match self.supervise_admitted_route(&snapshot, current) { - Ok(true) => { - self.clear_fault(&route_key); - progressed = true; - continue; - } - Ok(false) => self.clear_fault(&route_key), - Err(error) => self.report_once( - &route_key, - format!( - "Idunn rejected admitted {} route continuity: {error:#}", - current.value.target - ), - ), - } - let topology_key = format!("topology:{}", current.value.target); - match self.refresh_admitted_topology(&snapshot, current) { - Ok(true) => { - self.clear_fault(&topology_key); - progressed = true; - continue; - } - Ok(false) => self.clear_fault(&topology_key), - Err(error) => self.report_once( - &topology_key, - format!( - "Idunn preserved admitted {} after rejecting topology observation: {error:#}", - current.value.target - ), - ), - } continue; } let workload_error = operational_error @@ -7627,17 +7572,10 @@ impl Engine { // could not resolve. So a failed demotion mints nothing. The // failure is written to the generation, and the next attempt is // not before the deferral ends. - let demotion_key = format!("demotion:{}", current.value.target); - if demotion.is_ok() { - self.clear_fault(&demotion_key); - } if let Err(error) = demotion { - self.report_once( - &demotion_key, - format!( - "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", - current.value.target - ), + eprintln!( + "Idunn deferred continuity for admitted {} after refusing to demote its projection: {error:#}", + current.value.target ); let mut deferred = supervision.clone(); deferred.defer_continuity(now, &bounded_detail(&format!("{error:#}"))); @@ -7764,13 +7702,17 @@ impl Engine { "admitted route receipt names another incarnation" ); let binding = current.value.plan.parsed_inputs()?.1; - let lease_driver = if current.value.leasing.lease().is_some() { + let lease_driver = if let Some(lease) = current.value.leasing.lease() { let lease_path = &binding .process_write_lease .as_ref() .context("stateful admitted generation has no write-lease binding")? .record_path; let driver = CultCacheWriteLeaseDriver::new(¤t.value.target, lease_path); + ensure!( + driver.observe_exact(lease)?, + "admitted process write lease is no longer exact" + ); Some(driver) } else { None @@ -8935,11 +8877,10 @@ impl Engine { rollback_allowed, &gate, )?; - let observation = match self.prove_routed_candidate_through_handoff( + let observation = match self.prove_stable_route( &admitted.value, &driver, membership_sha256, - preflight, ) { Ok(observation) => observation, Err(proof_error) if !rollback_allowed => { @@ -9864,42 +9805,6 @@ impl Engine { ) } - /// Routing's proof, waiting out nginx's reload handoff. `systemctl reload` - /// only signals the master: the old workers keep answering the stable - /// listener until the new configuration is parsed and they are told to - /// quit, so Idunn's first challenge can reach the incumbent. An answer - /// that names the preflight's incumbent is therefore not-yet for - /// `ROUTE_RELOAD_HANDOFF`; it can delay and never admit (the id is read - /// unauthenticated, and the candidate's proof is checked as before). - /// Anything else fails the attempt at once. One challenge per poll and - /// nothing else: no install, rollback, route gate or persistence. - fn prove_routed_candidate_through_handoff( - &self, - transaction: &DeploymentTransaction, - driver: &NginxRouteDriver, - membership_sha256: String, - preflight: &crate::drivers::RoutePreflightReceipt, - ) -> Result { - let candidate = &required(&transaction.activation, "route activation")?.runtime_instance_id; - let until = handoff_now() + ROUTE_RELOAD_HANDOFF; - loop { - match self.prove_stable_route(transaction, driver, membership_sha256.clone()) { - Err(error) - if preflight.changes_route() - && handoff_now() < until - && error.downcast_ref::().is_some_and(|answer| { - preflight.incumbent_runtime_instance_id.as_deref() - == Some(answer.answered_instance.as_str()) - && answer.answered_instance != *candidate - }) => - { - handoff_sleep(ROUTE_RELOAD_HANDOFF_POLL); - } - result => return result, - } - } - } - fn prove_stable_route_against( &self, expected: &IdunnExpectedIncarnationRecord, @@ -9924,15 +9829,7 @@ impl Engine { challenged_at_unix_millis, received_at_unix_millis, &response.canonical_presence, - ) - .map_err(|error| match presence_answer_ids(&response.canonical_presence) { - Some((answered_instance, answered_activation_signer)) => error.context(RouteProofAnswer { - answered_instance: route_proof_label(&answered_instance), - answered_activation_signer: route_proof_label(&answered_activation_signer), - candidate_instance: route_proof_label(&activation.runtime_instance_id), - }), - None => error, - })?; + )?; let observation = RouteObservation { route_id: driver.binding.route_id.clone(), runtime_instance_id: activation.runtime_instance_id.clone(), @@ -11555,97 +11452,10 @@ fn route_proof_report_key(transaction_id: &str) -> String { format!("route-proof:{transaction_id}") } -/// How long Routing waits for nginx's old workers to let go of the stable -/// listener after a reload, derived from two ceilings: one Forward actuation -/// must suffice per attempt (12 an hour), and the 120 s Routing deadline must -/// keep room for at least four more backoff-spaced attempts if the route -/// genuinely never switches. -const ROUTE_RELOAD_HANDOFF: Duration = - if cfg!(test) { Duration::from_millis(400) } else { Duration::from_secs(10) }; -const ROUTE_RELOAD_HANDOFF_POLL: Duration = - if cfg!(test) { Duration::from_millis(20) } else { Duration::from_millis(250) }; - -/// The handoff loop's clock. Production reads the real one; a test may install -/// a virtual one on its own thread, so the window and the poll cadence are -/// measured exactly, whatever the host load. -#[cfg(not(test))] -fn handoff_now() -> std::time::Instant { - std::time::Instant::now() -} -#[cfg(not(test))] -fn handoff_sleep(pause: Duration) { - thread::sleep(pause); -} -#[cfg(test)] -thread_local! { - /// (virtual offset, sleeps taken, clock reads) while a test holds it. - static HANDOFF_CLOCK: std::cell::RefCell, usize)>> = - const { std::cell::RefCell::new(None) }; -} -#[cfg(test)] -fn handoff_now() -> std::time::Instant { - static BASE: std::sync::OnceLock = std::sync::OnceLock::new(); - let base = *BASE.get_or_init(std::time::Instant::now); - HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { - Some((offset, _, reads)) => { - *reads += 1; - assert!(*reads < 10_000, "the handoff loop spins without sleeping"); - base + *offset - } - None => std::time::Instant::now(), - }) -} -#[cfg(test)] -fn handoff_sleep(pause: Duration) { - HANDOFF_CLOCK.with(|clock| match clock.borrow_mut().as_mut() { - Some((offset, sleeps, _)) => { - *offset += pause; - sleeps.push(pause); - } - None => thread::sleep(pause), - }) -} - -/// Who answered a stable route challenge that failed authentication, as the -/// answer claimed itself: ids only, unauthenticated, never signature bytes. -/// A label on a rejection, and the one thing that may say "challenge again". -#[derive(Debug)] -struct RouteProofAnswer { - answered_instance: String, - answered_activation_signer: String, - candidate_instance: String, -} - -impl std::fmt::Display for RouteProofAnswer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "stable route answered as runtime instance {}, activation signer {} (unauthenticated); the candidate is {}", - self.answered_instance, self.answered_activation_signer, self.candidate_instance - ) - } -} - -/// The ids a presence claims, if it decodes at all. No re-encode check and no -/// validation: this labels a rejection and is never evidence. -fn presence_answer_ids(canonical: &[u8]) -> Option<(String, String)> { - let presence = - rmp_serde::from_slice::(canonical).ok()?; - Some((presence.runtime_instance_id, presence.activation_signer_identity_id)) -} - -/// An id cut to 96 bytes at a char boundary, control characters stripped. -fn route_proof_label(id: &str) -> String { - let id: String = id.chars().filter(|c| !c.is_control()).collect(); - id[..id.floor_char_boundary(96)].to_owned() -} - /// The stable route proof's failure in words: whether the listener said /// nothing or answered wrongly, then the specific refusal (HTTP status line, /// presence not active, challenge or authority disagreement). The texts it -/// carries name states and codes, never keys or signature bytes. A rejected -/// answer names the instance and activation signer it claimed, unauthenticated, -/// ids only. +/// carries name states and codes, never keys or signature bytes. fn route_proof_reason(error: &anyhow::Error) -> String { match error.downcast_ref::() { Some(ChallengeFailure::Silent(_)) => format!("silent, nothing answered: {error:#}"), @@ -14357,10 +14167,6 @@ mod tests { .replace( "/var/lib/gamecult/idunn/service-deployment-brake.cc", &world.root.join("service-deployment-brake.cc").display().to_string(), - ) - .replace( - "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", - &world.root.join("service-lifecycle-brake.cc").display().to_string(), ), )?; let recipe = RECIPE @@ -15264,132 +15070,6 @@ mod tests { odin_reports_ready(world, transaction, sequence) } - /// An Odin-correlated generation, admitted over `workload`, with the - /// sequence of the reading it holds. - fn odin_admitted_world( - workload: Arc, - ) -> Result<(EngineFixture, DeploymentTransaction, u64)> { - let world = EngineFixture::with_workload(workload)?; - let seeded = transaction_at(&world, DeploymentPhase::Fencing)?; - let first = seeded_sequence(None) + 1; - odin_reports_ready(&world, &seeded, first)?; - drive(&world, |transaction| transaction.phase == DeploymentPhase::Complete)?; - assert!(world.engine.retire_one_terminal_transaction()?); - Ok((world, seeded, first)) - } - - /// Odin's next reading, about an activation that is not the current one. - fn odin_republishes_another_activation( - world: &EngineFixture, - seeded: &DeploymentTransaction, - sequence: u64, - ) -> Result<()> { - let mut other = seeded.clone(); - other.activation.as_mut().context("no activation")?.runtime_instance_id = - sha256_id(b"another-incarnation"); - odin_republishes(world, &other, sequence) - } - - /// Engage the admitted target's lifecycle brake, at the path its seeded - /// binding names: a dead incarnation mints no continuity, so every later - /// tick reaches the same gates. - fn engage_lifecycle_brake_in(world: &EngineFixture) -> Result<()> { - let generation = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no admitted generation")? - .value - .clone(); - crate::provisioning::replace_lifecycle_brake( - &world.root.join("service-lifecycle-brake.cc"), - IdunnLifecycleBrakeRecord { - schema_version: IDUNN_LIFECYCLE_BRAKE_SCHEMA.into(), - authority: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_AUTHORITY.into(), - runtime_id: generation.expected.runtime_id.clone(), - target: "service".into(), - scope: cultnet_rs::IDUNN_LIFECYCLE_BRAKE_SCOPE.into(), - status: "engaged".into(), - reason: "test".into(), - updated_at_unix_millis: now_millis()?, - released_until_unix_millis: None, - }, - ) - } - - #[test] - fn a_dead_generation_does_not_take_odin_correlations() -> Result<()> { - let switch = SwitchWorkload::new(); - let (world, seeded, first) = odin_admitted_world(switch.clone())?; - switch.kill(); - // No continuity is minted, so no transaction ever owns the target and - // every tick reaches the topology gate. - engage_lifecycle_brake_in(&world)?; - let envelope = incumbent_envelope(&world)?; - - // A newer reading that binds the admitted activation, and one that does - // not: neither is taken, nor said, about an incarnation that is not there. - for reading in 0..2 { - if reading == 0 { - odin_republishes(&world, &seeded, first + 4)?; - } else { - odin_republishes_another_activation(&world, &seeded, first + 5)?; - } - for _ in 0..100 { - world.engine.supervise_one_admitted_generation()?; - } - assert!( - ControlSnapshot::read(&world.state_store)?.transactions.is_empty(), - "a transaction owns the target: later ticks never reach the gate" - ); - assert_eq!(incumbent_envelope(&world)?, envelope); - assert!( - !world.engine.fault_reports.lock().unwrap().contains_key("topology:service"), - "a dead generation reported a topology rejection" - ); - } - Ok(()) - } - - fn printed(world: &EngineFixture, key: &str) -> usize { - world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() - } - - #[test] - fn a_running_generation_with_a_non_binding_correlation_is_reported_once() -> Result<()> { - let (world, seeded, first) = odin_admitted_world(Arc::new(StillWorkload))?; - odin_republishes_another_activation(&world, &seeded, first + 4)?; - let envelope = incumbent_envelope(&world)?; - for _ in 0..50 { - assert!(!world.engine.supervise_one_admitted_generation()?); - } - assert_eq!(incumbent_envelope(&world)?, envelope); - // The engine said it once over fifty ticks. - assert_eq!(printed(&world, "topology:service"), 1); - assert!(world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); - - // A binding reading is adopted and ends the fault, so the key clears. - odin_republishes(&world, &seeded, first + 6)?; - assert!(world.engine.supervise_one_admitted_generation()?); - assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); - let adopted = ControlSnapshot::read(&world.state_store)? - .admitted_for("service") - .context("no generation")? - .value - .clone(); - assert_eq!(adopted.odin_publisher_sequence_cursor, first + 6); - - // A recurrence is said again, once; a reading withdrawn ends it too. - odin_republishes_another_activation(&world, &seeded, first + 8)?; - for _ in 0..20 { - world.engine.supervise_one_admitted_generation()?; - } - assert_eq!(printed(&world, "topology:service"), 2); - let store = SingleFileMessagePackBackingStore::new(&world.engine.options.odin_correlation_store); - assert!(store.delete_batch_if_unchanged(&store.pull_all_read_only_snapshot()?)?); - assert!(!world.engine.supervise_one_admitted_generation()?); - assert!(!world.engine.fault_reports.lock().unwrap().contains_key("topology:service")); - Ok(()) - } - fn ready_sequence(transaction: &DeploymentTransaction) -> Option { transaction .ready @@ -21978,25 +21658,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } - #[test] - fn status_says_how_overdue_a_route_challenge_is_and_claims_no_cause() -> Result<()> { - let now = 10_000_000_u64; - let (_, mut generation) = fixture_generation(FIXTURE_GENERATION)?; - generation.route_supervision = Some(RouteSupervisionState { - last_challenge_at_unix_millis: Some(now - 5_000), - next_challenge_at_unix_millis: Some(now - 4_000), - ..RouteSupervisionState::default() - }); - let lines = render_supervision("service", Some(&generation), None, now).join("\n"); - assert!(lines.contains("last-challenge-at 9995000 (5000 ms ago)"), "{lines}"); - assert!(lines.contains("next-challenge-at 9996000 (past due 4000 ms)"), "{lines}"); - assert!(!lines.contains("waits on"), "{lines}"); - generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(now); - let lines = render_supervision("service", Some(&generation), None, now).join("\n"); - assert!(!lines.contains("past due"), "{lines}"); - Ok(()) - } - #[test] fn status_shows_each_targets_meters_including_a_target_with_no_generation() -> Result<()> { let now = 10_000_000_u64; @@ -22096,11 +21757,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation: IdunnRuntimeActivationRecord, provider: cultnet_rs::ServiceIdentitySigner, activation_signer: cultnet_rs::IdunnRuntimeActivationSigner, - /// A second key the transaction's activation record never named. - other_signer: cultnet_rs::IdunnRuntimeActivationSigner, - /// The instance the stable listener answered as before the reload. - incumbent_instance: Mutex>, - stable_answers: AtomicUsize, state: Mutex<&'static str>, capacity: AtomicU32, sequence: AtomicU64, @@ -22130,36 +21786,11 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide /// A first Odin candidate answering its Warming challenge: the /// detail names the challenge as `idunn-warming:`. FirstOdinWarming, - /// The stable listener's first n answers come from the incumbent - /// (its instance id, a key the record never named), then Honest. - IncumbentFirst(usize), - /// The candidate's own instance id, signed with the wrong key. - ForeignActivation, - /// Neither the candidate nor the incumbent, wrong key. - StrangerInstance, } impl RuntimeStub { fn answer(&self, message_id: &str) -> Result> { - self.answer_on(message_id, false) - } - - fn answer_on(&self, message_id: &str, stable: bool) -> Result> { let expected = &self.expected; - let own = self.activation.runtime_instance_id.clone(); - let (instance, signer) = match *self.reply.lock().unwrap() { - Reply::IncumbentFirst(n) - if stable && self.stable_answers.fetch_add(1, Ordering::SeqCst) < n => - { - let incumbent = self.incumbent_instance.lock().unwrap().clone(); - (incumbent.context("no incumbent instance set")?, &self.other_signer) - } - Reply::ForeignActivation if stable => (own, &self.other_signer), - Reply::StrangerInstance if stable => { - (sha256_id(b"stranger instance"), &self.other_signer) - } - _ => (own, &self.activation_signer), - }; let mut presence = cultnet_rs::GameCultRuntimePresenceHealthRecord { schema_version: cultnet_rs::GAMECULT_RUNTIME_PRESENCE_HEALTH_SCHEMA.into(), target: expected.target.clone(), @@ -22171,7 +21802,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide state_schema_generation: expected.state_schema_generation.clone(), state_contract_sha256: expected.state_contract_sha256.clone(), runtime_id: expected.runtime_id.clone(), - runtime_instance_id: instance, + runtime_instance_id: self.activation.runtime_instance_id.clone(), bound_endpoint: expected .route .as_ref() @@ -22198,7 +21829,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide observed_at_unix_millis: now_millis()?, signature_algorithm: "ed25519".into(), signature: Vec::new(), - activation_signer_identity_id: signer.identity_id(), + activation_signer_identity_id: self.activation_signer.identity_id(), activation_signature: Vec::new(), }; let proof = presence.canonical_proof_payload()?; @@ -22206,7 +21837,8 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .provider .sign::(&proof) .signature; - presence.activation_signature = signer.sign_presence_proof(&presence)?; + presence.activation_signature = + self.activation_signer.sign_presence_proof(&presence)?; Ok(rmp_serde::to_vec(&presence)?) } @@ -22214,7 +21846,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide *self.state.lock().unwrap() = state; } - fn serve_one(&self, mut stream: TcpStream, stable: bool) -> Result<()> { + fn serve_one(&self, mut stream: TcpStream) -> Result<()> { stream.set_read_timeout(Some(Duration::from_secs(3)))?; let mut request = Vec::new(); let mut chunk = [0_u8; 4096]; @@ -22261,7 +21893,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide record_key: self.expected.target.clone(), stored_at: rfc3339_millis(now_millis()?)?, payload_encoding: cultnet_rs::CultNetRawPayloadEncoding::Messagepack, - payload: self.answer_on(&message_id, stable)?, + payload: self.answer(&message_id)?, source_runtime_id: None, source_agent_id: None, source_role: None, @@ -22303,7 +21935,7 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide hook(); } if !stub.hang_up.load(Ordering::SeqCst) { - let _ = stub.serve_one(stream, !candidate); + let _ = stub.serve_one(stream); } } Err(_) => std::thread::sleep(Duration::from_millis(5)), @@ -22415,10 +22047,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide .replace( "/etc/nginx/idunn-stream-routes/service.conf", &config.display().to_string().replace('\\', "/"), - ) - .replace( - "/var/lib/gamecult/idunn/service-lifecycle-brake.cc", - &world.root.join("service-lifecycle-brake.cc").display().to_string(), ); let (binding, recipe_source) = if stateful { std::fs::create_dir_all(world.root.join("lease"))?; @@ -22523,14 +22151,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide )?; let mut seed = Vec::new(); let activation = launch.write_credential(&mut seed)?; - let mut other_seed = Vec::new(); - IdunnRuntimeActivationLaunch::issue( - &expected, - sha256_id(b"unrecorded activation"), - now, - &world.engine.idunn_signer, - )? - .write_credential(&mut other_seed)?; let recorded = fixture_transaction(FIXTURE_TRANSACTIONS[0].1)?.1; let mut workload = recorded.workload.clone().context("no recorded workload")?; match &mut workload { @@ -22562,11 +22182,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide activation_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( &seed[..], )?, - other_signer: cultnet_rs::IdunnRuntimeActivationSigner::from_credential_reader( - &other_seed[..], - )?, - incumbent_instance: Mutex::new(None), - stable_answers: AtomicUsize::new(0), state: Mutex::new("warming"), capacity: AtomicU32::new(minimum_capacity), sequence: AtomicU64::new(0), @@ -24546,325 +24161,30 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide #[cfg(unix)] #[test] - fn a_dead_unit_is_restarted_without_a_route_challenge() -> Result<()> { + fn a_failed_challenge_and_a_dead_unit_in_one_tick_do_not_lose_a_cas() -> Result<()> { let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; routed.admit()?; routed.workload.kill(); routed.stub.hang_up.store(true, Ordering::SeqCst); routed.drift()?; - let before = admitted(&routed)?.route_supervision; - let reloads = routed.reloads(); - let survivals = meters_of(&routed.world, "service")?.route_survivals; - // Tick 1: restart, not route repair, is the response to a dead - // incarnation. The continuity is minted and no route survival is - // charged for a proof nobody could have passed. + // Tick 1: the repair charges the target's meters and the proof + // fails. Both are writes, so they end the pass: nothing else is + // decided from the snapshot they made stale. + assert!(routed.tick()?); + let state = admitted(&routed)?.route_supervision.context("no route state")?; + assert_eq!(state.consecutive_failures, 1); + assert!(state.degraded_since_unix_millis.is_some()); + assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); + let meters = meters_of(&routed.world, "service")?; + assert_eq!(meters.route_survivals.len(), 2); + assert!(meters.route_actuations.is_empty()); + + // Tick 2: the route waits, and the dead unit is restarted from a + // fresh read, counted in the same write that mints it. assert!(routed.tick()?); assert_eq!(ControlSnapshot::read(&routed.world.state_store)?.transactions.len(), 1); assert_eq!(meters_of(&routed.world, "service")?.continuity_restarts.len(), 1); - assert_eq!(meters_of(&routed.world, "service")?.route_survivals, survivals); - assert_eq!(admitted(&routed)?.route_supervision, before); - assert_eq!(routed.reloads(), reloads, "the dead unit's route was repaired"); - Ok(()) - } - - /// A target with a process write lease, admitted and running. - #[cfg(unix)] - fn admitted_stateful_world() -> Result { - let routed = stateful_routed_world(Odin::Unreachable, DeploymentPhase::Fencing)?; - routed.drive_until(|t| t.phase == DeploymentPhase::AwaitingReady)?; - let granted_lease = routed - .transaction()? - .leasing - .as_ref() - .and_then(LeasingEvidence::lease_sha256) - .context("no granted lease")? - .to_owned(); - routed.stub.set_state("active"); - *routed.stub.lease.lock().unwrap() = Some(granted_lease); - routed.step()?; - routed.drive_until(|t| t.phase == DeploymentPhase::Routing)?; - routed.promote()?; - routed.drive_until(|transaction| transaction.is_terminal())?; - assert!(routed.world.engine.retire_one_terminal_transaction()?); - Ok(routed) - } - - /// Engage the target's lifecycle brake: the dead path may not mint. - fn engage_lifecycle_brake(routed: &RoutedWorld) -> Result<()> { - engage_lifecycle_brake_in(&routed.world) - } - #[cfg(unix)] - #[test] - fn a_parked_routed_target_is_not_route_supervised() -> Result<()> { - let routed = admitted_stateful_world()?; - let generation = admitted(&routed)?; - let lease = generation.leasing.lease().context("no lease")?.clone(); - CultCacheWriteLeaseDriver::new( - "service", - routed.world.root.join("lease/process-write-lease.cc"), - ) - .revoke_exact(Some(&lease))?; - routed.workload.kill(); - engage_lifecycle_brake(&routed)?; - routed.drift()?; - let envelope = incumbent_envelope(&routed.world)?; - let reloads = routed.reloads(); - let programs = routed.world.route_stubs.count(""); - - for _ in 0..100 { - routed.tick()?; - } - assert_eq!(routed.reloads(), reloads, "a parked target's route was reloaded"); - assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); - assert_eq!(incumbent_envelope(&routed.world)?, envelope); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - let reports = routed.world.engine.fault_reports.lock().unwrap(); - assert!( - !reports.keys().any(|key| key.starts_with("route-supervision:") || key.starts_with("route:")), - "{:?}", - reports.keys().collect::>() - ); - Ok(()) - } - - fn fault_keys(routed: &RoutedWorld) -> Vec { - routed.world.engine.fault_reports.lock().unwrap().keys().cloned().collect() - } - - /// Soul's probe: a route record that cannot be written leaves the - /// challenge past due on a running incarnation. Status shows the age - /// and does not name a cause it cannot see. - #[cfg(unix)] - #[test] - fn status_of_a_past_due_route_on_a_running_incarnation_claims_no_wait() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.stub.hang_up.store(true, Ordering::SeqCst); - routed.drift()?; - assert!(routed.tick()?); - assert!(admitted(&routed)?.route_supervision.unwrap().degraded_since_unix_millis.is_some()); - routed.stub.hang_up.store(false, Ordering::SeqCst); - edit_incumbent(&routed.world, |generation| { - generation.route_supervision.as_mut().unwrap().next_challenge_at_unix_millis = Some(1); - })?; - let fragment = routed.fragment_path(); - std::fs::remove_file(&fragment)?; - std::fs::create_dir(&fragment)?; - for _ in 0..20 { - routed.tick()?; - } - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - assert!(fault_keys(&routed).contains(&"route-supervision:service".to_owned())); - let generation = admitted(&routed)?; - let lines = render_supervision("service", Some(&generation), None, now_millis()?).join("\n"); - assert!(lines.contains("(past due "), "{lines}"); - assert!(!lines.contains("waits on"), "{lines}"); - Ok(()) - } - - /// A target whose lease store is empty while its process runs is a - /// dead-path case: the route is not challenged for it. - #[cfg(unix)] - #[test] - fn a_running_target_with_an_empty_lease_is_not_route_challenged() -> Result<()> { - let routed = admitted_stateful_world()?; - let lease = admitted(&routed)?.leasing.lease().context("no lease")?.clone(); - CultCacheWriteLeaseDriver::new( - "service", - routed.world.root.join("lease/process-write-lease.cc"), - ) - .revoke_exact(Some(&lease))?; - engage_lifecycle_brake(&routed)?; - routed.drift()?; - let envelope = incumbent_envelope(&routed.world)?; - let reloads = routed.reloads(); - let programs = routed.world.route_stubs.count(""); - for _ in 0..50 { - routed.tick()?; - } - assert_eq!(routed.reloads(), reloads, "an empty-lease target's route was reloaded"); - assert_eq!(routed.world.route_stubs.count(""), programs, "a program ran"); - assert_eq!(incumbent_envelope(&routed.world)?, envelope); - assert!(ControlSnapshot::read(&routed.world.state_store)?.transactions.is_empty()); - assert!( - !fault_keys(&routed).iter().any(|key| key.starts_with("route")), - "{:?}", - fault_keys(&routed) - ); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_host_that_answers_with_a_death_ends_the_silence_fault() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - engage_lifecycle_brake(&routed)?; - routed.workload.silence(true); - for _ in 0..5 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, "observe:service"), 1); - routed.workload.silence(false); - routed.workload.kill(); - routed.tick()?; - assert!(!fault_keys(&routed).contains(&"observe:service".to_owned())); - Ok(()) - } - - /// Replace the topology store with a directory: every read of it fails. - #[cfg(unix)] - fn break_topology(routed: &RoutedWorld) -> Result<()> { - let store = routed.world.root.join("topology.cc"); - if store.exists() { - std::fs::rename(&store, routed.world.root.join("topology.aside"))?; - } - std::fs::create_dir(&store)?; - Ok(()) - } - - #[cfg(unix)] - fn mend_topology(routed: &RoutedWorld) -> Result<()> { - let store = routed.world.root.join("topology.cc"); - std::fs::remove_dir(&store)?; - let aside = routed.world.root.join("topology.aside"); - if aside.exists() { - std::fs::rename(aside, &store)?; - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_stale_incarnation_and_a_projection_repair_that_fail_are_each_said_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let keys = ["stale-incarnation:service", "projection-repair:service"]; - for round in 1..=2 { - break_topology(&routed)?; - for _ in 0..30 { - routed.tick()?; - } - for key in keys { - assert_eq!(printed_in(&routed, key), round, "{key}"); - assert!(fault_keys(&routed).contains(&key.to_owned()), "{key}"); - } - mend_topology(&routed)?; - routed.tick()?; - for key in keys { - assert!(!fault_keys(&routed).contains(&key.to_owned()), "{key}"); - } - } - Ok(()) - } - - #[cfg(unix)] - #[test] - fn a_refused_demotion_is_said_once_and_cleared_when_it_succeeds() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.workload.kill(); - break_topology(&routed)?; - // Each attempt is let through by forgetting the deferral it wrote. - for _ in 0..6 { - set_meters(&routed.world, &TargetSupervision::new("service"))?; - routed.tick()?; - } - assert_eq!(printed_in(&routed, "demotion:service"), 1); - assert!(fault_keys(&routed).contains(&"demotion:service".to_owned())); - mend_topology(&routed)?; - set_meters(&routed.world, &TargetSupervision::new("service"))?; - routed.tick()?; - assert!(!fault_keys(&routed).contains(&"demotion:service".to_owned())); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn an_unsafe_write_lease_store_is_said_once_and_cleared_when_it_is_exact() -> Result<()> { - let routed = admitted_stateful_world()?; - let store = routed.world.root.join("lease/process-write-lease.cc"); - let exact = std::fs::read(&store)?; - for round in 1..=2 { - std::fs::write(&store, b"not a lease store")?; - for _ in 0..30 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, "write-lease:service"), round); - assert!(fault_keys(&routed).contains(&"write-lease:service".to_owned())); - std::fs::write(&store, &exact)?; - routed.tick()?; - assert!(!fault_keys(&routed).contains(&"write-lease:service".to_owned())); - } - Ok(()) - } - - fn printed_in(routed: &RoutedWorld, key: &str) -> usize { - routed.world.engine.printed_reports.lock().unwrap().iter().filter(|printed| *printed == key).count() - } - - #[cfg(unix)] - #[test] - fn a_running_route_whose_fragment_cannot_be_read_is_reported_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - let key = "route-supervision:service"; - let fragment = routed.fragment_path(); - let bytes = std::fs::read(&fragment)?; - // The fragment is a directory: reading it fails, and nothing is - // written, so every tick meets the same error. - std::fs::remove_file(&fragment)?; - std::fs::create_dir(&fragment)?; - for _ in 0..30 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, key), 1); - assert!(routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); - - // The fragment is back: the fault ends and the key clears. - std::fs::remove_dir(&fragment)?; - std::fs::write(&fragment, &bytes)?; - routed.tick()?; - assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key(key)); - - // A recurrence is said again, once. - std::fs::remove_file(&fragment)?; - std::fs::create_dir(&fragment)?; - for _ in 0..30 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, key), 2); - Ok(()) - } - - /// Skipping route and topology on a tick the host could not be asked is - /// the steward's ruling (no activation can be bound without an - /// observation; no routed target runs on a host actuator today). - #[cfg(unix)] - #[test] - fn a_host_that_cannot_be_asked_skips_route_supervision_and_is_said_once() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - routed.admit()?; - routed.drift()?; - let envelope = incumbent_envelope(&routed.world)?; - let reloads = routed.reloads(); - routed.workload.silence(true); - for _ in 0..30 { - routed.tick()?; - } - assert_eq!(routed.reloads(), reloads, "route supervision ran without an observation"); - assert_eq!(incumbent_envelope(&routed.world)?, envelope); - assert_eq!(printed_in(&routed, "observe:service"), 1); - - routed.workload.silence(false); - routed.tick()?; - assert!(!routed.world.engine.fault_reports.lock().unwrap().contains_key("observe:service")); - routed.workload.silence(true); - for _ in 0..5 { - routed.tick()?; - } - assert_eq!(printed_in(&routed, "observe:service"), 2); Ok(()) } @@ -25108,199 +24428,6 @@ GAMECULT_RUNTIME_PRESENCE_IDENTITY = "/etc/gamecult/service/runtime-presence-ide Ok(()) } - // ----------------------------------------------------------------- - // The reload handoff: nginx's old workers may answer the stable - // listener for a moment after `systemctl reload` returns. - // ----------------------------------------------------------------- - - /// A deploy world at Routing whose preflight found an incumbent route - /// (another instance, another membership), so the install changes it. - #[cfg(unix)] - fn handoff_world(reply: Reply) -> Result<(RoutedWorld, String)> { - handoff_world_with(reply, false) - } - - /// `same_membership`: the incumbent's membership equals the - /// candidate's, so the install changes no route. - #[cfg(unix)] - fn handoff_world_with(reply: Reply, same_membership: bool) -> Result<(RoutedWorld, String)> { - let routed = routed_deploy_world(Odin::Unreachable, 1, DeploymentPhase::Fencing)?; - let incumbent = sha256_id(b"incumbent instance"); - let current = resident(&routed.world)?; - let mut next = current.value.clone(); - let mut on_disk = b"incumbent route\n".to_vec(); - if same_membership { - on_disk = NginxRouteDriver::new( - next.plan.as_ref().unwrap().parsed_inputs()?.1.route.context("no route binding")?, - ) - .render(next.expected.as_ref().context("no Expected")?)?; - } - let preflight = next.route_preflight.as_mut().context("no preflight")?; - preflight.incumbent_runtime_instance_id = Some(incumbent.clone()); - preflight.incumbent_membership_sha256 = Some(sha256_id(&on_disk)); - preflight.incumbent_configuration = Some(on_disk.clone()); - replace_transaction(&routed.world.state_store, ¤t, &next)?; - std::fs::write(routed.fragment_path(), &on_disk)?; - routed.stub.set_state("active"); - routed.run_to_routing()?; - *routed.stub.incumbent_instance.lock().unwrap() = Some(incumbent.clone()); - *routed.stub.reply.lock().unwrap() = reply; - Ok((routed, incumbent)) - } - - fn forwards(routed: &RoutedWorld) -> Result { - Ok(meters_of(&routed.world, "service")?.route_actuations.len()) - } - - fn proof_report(routed: &RoutedWorld) -> Result> { - let key = route_proof_report_key(&routed.transaction()?.transaction_id); - Ok(routed - .world - .engine - .fault_reports - .lock() - .unwrap() - .get(&key) - .and_then(|report| report.last.lock().unwrap().clone())) - } - - /// Holds a virtual handoff clock on this thread: the wait is measured - /// in the loop's own sleeps, so no host load can move it. - struct VirtualHandoffClock; - - impl VirtualHandoffClock { - fn hold() -> Self { - HANDOFF_CLOCK.with(|c| *c.borrow_mut() = Some((Duration::ZERO, Vec::new(), 0))); - Self - } - - /// The pauses the loop took so far. - fn sleeps(&self) -> Vec { - HANDOFF_CLOCK.with(|c| c.borrow().as_ref().unwrap().1.clone()) - } - } - - impl Drop for VirtualHandoffClock { - fn drop(&mut self) { - HANDOFF_CLOCK.with(|c| *c.borrow_mut() = None); - } - } - - #[cfg(unix)] - #[test] - fn routing_waits_out_the_incumbent_answering_after_reload() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, _) = handoff_world(Reply::IncumbentFirst(3))?; - let before = routed.reloads(); - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - routed.promote()?; - assert_eq!(routed.reloads() - before, 1, "an install and no rollback"); - assert_eq!(forwards(&routed)?, 1); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 4); - assert_eq!(clock.sleeps(), vec![ROUTE_RELOAD_HANDOFF_POLL; 3]); - assert!(proof_report(&routed)?.is_none(), "a route proof report remains"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn routing_fails_once_when_the_incumbent_never_lets_go() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, incumbent) = handoff_world(Reply::IncumbentFirst(usize::MAX))?; - let before = routed.reloads(); - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - let error = routed.step().unwrap_err(); - // The window is exactly ROUTE_RELOAD_HANDOFF of polls at exactly - // the poll cadence: one challenge before each pause and one after - // the last. - let sleeps = clock.sleeps(); - let polls = (ROUTE_RELOAD_HANDOFF.as_millis() / ROUTE_RELOAD_HANDOFF_POLL.as_millis()) as usize; - assert_eq!(sleeps, vec![ROUTE_RELOAD_HANDOFF_POLL; polls]); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, polls + 1); - assert_eq!(routed.reloads() - before, 2, "install and one rollback"); - assert_eq!(forwards(&routed)?, 1); - let said = proof_report(&routed)?.context("no route proof report")?; - let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; - let signer = routed.stub.other_signer.identity_id(); - for text in [format!("{error:#}"), said] { - assert!(text.contains(&format!("runtime instance {incumbent}")), "{text}"); - assert!(text.contains(&format!("activation signer {signer}")), "{text}"); - assert!(text.contains(&format!("the candidate is {candidate}")), "{text}"); - } - routed.step().unwrap_err(); - assert_eq!(forwards(&routed)?, 2, "the second attempt charges one more Forward"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn routing_fails_at_once_when_the_candidate_answers_with_a_foreign_activation() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, _) = handoff_world(Reply::ForeignActivation)?; - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - let error = routed.step().unwrap_err(); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); - assert!(clock.sleeps().is_empty()); - let text = format!("{error:#}"); - let candidate = routed.transaction()?.activation.unwrap().runtime_instance_id; - assert!(text.contains("runtime activation proof verification failed"), "{text}"); - assert!(text.contains(&format!("runtime instance {candidate}")), "{text}"); - Ok(()) - } - - #[cfg(unix)] - #[test] - fn routing_fails_at_once_for_a_stranger_instance() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, _) = handoff_world(Reply::StrangerInstance)?; - let before = routed.reloads(); - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - routed.step().unwrap_err(); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); - assert!(clock.sleeps().is_empty()); - assert_eq!(routed.reloads() - before, 2); - Ok(()) - } - - /// An incumbent whose membership equals the candidate's has no route - /// to hand off, so its answer is a plain failure, never waited on. - #[cfg(unix)] - #[test] - fn routing_does_not_wait_when_the_route_is_unchanged() -> Result<()> { - let clock = VirtualHandoffClock::hold(); - let (routed, _) = handoff_world_with(Reply::IncumbentFirst(1), true)?; - let hits = routed.stub.stable_hits.load(Ordering::SeqCst); - routed.step().unwrap_err(); - assert_eq!(routed.stub.stable_hits.load(Ordering::SeqCst) - hits, 1); - assert!(clock.sleeps().is_empty()); - Ok(()) - } - - #[test] - fn route_proof_answer_names_ids_only() -> Result<()> { - let routed = routed_world(Odin::Unreachable, 1, DeploymentPhase::Fencing, false)?; - let mut presence = rmp_serde::from_slice::( - &routed.stub.answer("probe")?, - )?; - presence.runtime_instance_id = format!("a\u{7}{}", "é".repeat(5_000)); - presence.activation_signer_identity_id = "b".repeat(10_240); - presence.signature = vec![0xAB; 64]; - presence.activation_signature = vec![0xCD; 64]; - let (instance, signer) = presence_answer_ids(&rmp_serde::to_vec(&presence)?) - .context("a decodable presence gave no ids")?; - let text = RouteProofAnswer { - answered_instance: route_proof_label(&instance), - answered_activation_signer: route_proof_label(&signer), - candidate_instance: route_proof_label(&"c".repeat(10_240)), - } - .to_string(); - assert!(text.len() <= 400, "{} bytes", text.len()); - assert!(!text.contains('\u{7}')); - assert!(!text.contains("abab") && !text.contains("cdcd")); - assert!(presence_answer_ids(b"not msgpack presence").is_none()); - Ok(()) - } - /// The live shape of F1. A continuity reuses the admitted Expected, so /// its membership is the admitted fragment already on disk: while its /// proof fails, the install and the rollback write nothing and reload diff --git a/src/drivers.rs b/src/drivers.rs index 7dfeaee..46cb916 100644 --- a/src/drivers.rs +++ b/src/drivers.rs @@ -762,7 +762,7 @@ impl StaticReleasePort for FilesystemStaticRelease { binding: &StaticReleaseBinding, release_id: &str, ) -> Result { - use std::os::unix::fs::{OpenOptionsExt, PermissionsExt, symlink}; + use std::os::unix::fs::{OpenOptionsExt, symlink}; // Everything up to and including the rename can refuse, and only that // refuses: the link has not moved when `FlipRefused` is returned. @@ -823,10 +823,6 @@ impl StaticReleasePort for FilesystemStaticRelease { .create_new(true) .mode(0o444) .open(&temporary_carry)?; - // The creation mode is masked by the service's umask - // (UMask=0027 yields 0440), and the web server reads this file - // as "other"; say 0444 on the open handle. - file.set_permissions(fs::Permissions::from_mode(0o444))?; file.write_all(&bytes)?; file.sync_all()?; fs::rename(&temporary_carry, &carried)?; @@ -17049,43 +17045,6 @@ processes 9 Ok(()) } - /// The service runs with UMask=0027, which masks the create mode to 0440 - /// and hides the carry from the web user. The umask is process-global, so - /// the proof runs in a child copy of this test binary and no sibling test - /// ever sees it. - #[cfg(unix)] - #[test] - fn static_release_carry_is_0444_under_the_service_umask() -> Result<()> { - use std::os::unix::fs::PermissionsExt; - const CHILD: &str = "IDUNN_TEST_CARRY_UMASK_CHILD"; - const NAME: &str = "static_release_carry_is_0444_under_the_service_umask"; - if std::env::var_os(CHILD).is_none() { - let path = format!("{}::{NAME}", module_path!().split_once("::").map_or("", |(_, rest)| rest)); - let output = std::process::Command::new(std::env::current_exe()?) - .args(["--exact", path.as_str(), "--nocapture", "--test-threads=1"]) - .env(CHILD, "1") - .output()?; - assert!( - output.status.success(), - "child under umask 0027 failed:\n{}\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - assert!( - String::from_utf8_lossy(&output.stdout).contains("1 passed"), - "the child filter matched no test:\n{}", - String::from_utf8_lossy(&output.stdout) - ); - return Ok(()); - } - unsafe { libc::umask(0o027) }; - let site = stand_in_site()?; - FilesystemStaticRelease.carry_and_flip(&site.binding, RELEASE_A)?; - let carried = site.binding.current_link.join("config.json"); - assert_eq!(fs::metadata(&carried)?.permissions().mode() & 0o777, 0o444); - Ok(()) - } - #[cfg(unix)] #[test] fn static_release_missing_site_config_fails_before_the_link_moves() -> Result<()> { diff --git a/src/provisioning.rs b/src/provisioning.rs index 324785e..5af3903 100644 --- a/src/provisioning.rs +++ b/src/provisioning.rs @@ -304,7 +304,7 @@ fn lifecycle_brake_status(options: &BTreeMap) -> Result<()> { Ok(()) } -pub(crate) fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { +fn replace_lifecycle_brake(path: &Path, record: IdunnLifecycleBrakeRecord) -> Result<()> { record.validate()?; let store = SingleFileMessagePackBackingStore::new(path); let entries = store.pull_all_read_only_snapshot()?;